Julia: Function vs. Macro

using DataFrames, Benchmark

# FUNCTION TO READ CSV FILE TO DATAFRAME
function csv2df(file)
  d = readcsv(file, header = true)
  head = d[2]
  data = d[1]
  dt = DataFrame()
  for i in 1:size(head)[2]
    dt[symbol(head[i])] = data[:, i]
  end
  return dt
end

df_funct() = csv2df("survdt.csv");

# MACRO TO READ CSV FILE TO DATAFRAME
macro csv2df(file)
  d = readcsv(file, header = true)
  head = d[2]
  data = d[1]
  dt = DataFrame()
  for i in 1:size(head)[2]
    dt[symbol(head[i])] = data[:, i]
  end
  return dt
end

df_macro() = @csv2df "survdt.csv";

# FUNCTION IN THE DATAFRAME PACKAGE
df_frame() = readtable("survdt.csv");

print(compare([df_funct, df_macro, df_frame], 10))

# | Row | Function   | Average  | Relative  | Replications |
# |-----|------------|----------|-----------|--------------|
# | 1   | "df_funct" | 0.164968 | 4.07328e6 | 10           |
# | 2   | "df_macro" | 4.05e-8  | 1.0       | 10           |
# | 3   | "df_frame" | 0.063254 | 1.56183e6 | 10           |
Advertisements

Fitting Lasso with Julia

Julia Code

using RDatasets, DataFrames, GLMNet

data = dataset("MASS", "Boston");
y = array(data[:, 14]);
x = array(data[:, 1:13]);

cv = glmnetcv(x, y);
cv.path.betas[:, indmin(cv.meanloss)];
result = DataFrame();
result[:Vars] = names(data)[1:13];
result[:Beta] = cv.path.betas[:, indmin(cv.meanloss)];
result

# | Row | Vars    | Beta       |
# |-----|---------|------------|
# | 1   | Crim    | -0.0983463 |
# | 2   | Zn      | 0.0414416  |
# | 3   | Indus   | 0.0        |
# | 4   | Chas    | 2.68519    |
# | 5   | NOx     | -16.3066   |
# | 6   | Rm      | 3.86694    |
# | 7   | Age     | 0.0        |
# | 8   | Dis     | -1.39602   |
# | 9   | Rad     | 0.252687   |
# | 10  | Tax     | -0.0098268 |
# | 11  | PTRatio | -0.929989  |
# | 12  | Black   | 0.00902588 |
# | 13  | LStat   | -0.5225    |

R Code

library(glmnet)
data(Boston, package = "MASS")

x <- as.matrix(Boston[, 1:13])
y <- as.matrix(Boston[, 14])

cv <- cv.glmnet(x, y, nfolds = 10) 	
mdl <- glmnet(x, y, lambda = cv$lambda.min)
mdl$beta

# crim     -0.098693203
# zn        0.041588291
# indus     .          
# chas      2.681633344
# nox     -16.354590598
# rm        3.860035926
# age       .          
# dis      -1.399697121
# rad       0.255484621
# tax      -0.009935509
# ptratio  -0.931031828
# black     0.009031422
# lstat    -0.522741592

Estimating GLM with Julia

using DataFrames, GLM

df1 = readtable("credit_count.txt");

df2 = df1[df1[:CARDHLDR] .== 1, [:DEFAULT, :MAJORDRG, :MINORDRG, :INCOME, :OWNRENT]];

mdl = glm(DEFAULT ~ MAJORDRG + MINORDRG + INCOME + OWNRENT, df2, Binomial());

print(mdl);
# Coefficients:
#                  Estimate  Std.Error  z value Pr(>|z|)
# (Intercept)      -1.20444  0.0908218 -13.2616  < eps()
# MAJORDRG         0.203135  0.0692537  2.93319   0.0034
# MINORDRG         0.202727  0.0479741  4.22575   2.4e-5
# INCOME       -0.000442229 4.04222e-5 -10.9402  < eps()
# OWNRENT         -0.201223  0.0716217 -2.80953    0.005

print(deviance(mdl));
# 6376.220859525586

Julia and SQLite

Similar to R and Pandas in Python, Julia provides a simple yet efficient interface with SQLite database. In addition, it is extremely handy to use sqldf() function, which is almost identical to the sqldf package in R, in SQLite package for data munging.

julia> # LOADING SQLITE PACKAGE

julia> using SQLite

julia> # CONNECT TO THE SQLITE DB FILE 

julia> db = SQLite.connect("/home/liuwensui/Documents/db/sqlitedb/csdata.db")

julia> # SHOW TABLES IN THE DB 

julia> query("select name from sqlite_master where type = 'table'")
1x1 DataFrame
|-------|-----------|
| Row # | name      |
| 1     | tblcsdata |

julia> # PULL DATA FROM THE TABLE

julia> # THE DATA WOULD BE AUTOMATICALLY SAVED AS A DATAFRAME

julia> df1 = query("select * from tblcsdata");

julia> head(df1, 2)
6x12 DataFrame
|-------|---------|----------|-----------|---------|-----------|----------|-----|-----------|-------|-------|-------|-------|
| Row # | LEV_LT3 | TAX_NDEB | COLLAT1   | SIZE1   | PROF2     | GROWTH2  | AGE | LIQ       | IND2A | IND3A | IND4A | IND5A |
| 1     | 0.0     | 0.530298 | 0.0791719 | 13.132  | 0.0820164 | 1.16649  | 53  | 0.385779  | 0     | 0     | 1     | 0     |
| 2     | 0.0     | 0.370025 | 0.0407454 | 12.1326 | 0.0826154 | 11.092   | 54  | 0.224123  | 1     | 0     | 0     | 0     |

julia> # SELECT DATA FROM THE TABLE WITH SQLDF() FUNCTION 

julia> df2 = sqldf("select * from df1 where AGE between 25 and 30");

julia> # SUMMARIZE DATA WITH SQLDF() FUNCTION 

julia> df3 = sqldf("select age, avg(LEV_LT3) as avg_lev from df2 group by age")
6x2 DataFrame
|-------|-----|-----------|
| Row # | AGE | avg_lev   |
| 1     | 25  | 0.0923202 |
| 2     | 26  | 0.0915009 |
| 3     | 27  | 0.0579876 |
| 4     | 28  | 0.104191  |
| 5     | 29  | 0.0764582 |
| 6     | 30  | 0.0806471 |

Test Drive of Julia

   _       _ _(_)_     |  A fresh approach to technical computing
  (_)     | (_) (_)    |  Documentation: http://docs.julialang.org
   _ _   _| |_  __ _   |  Type "help()" to list help topics
  | | | | | | |/ _` |  |
  | | |_| | | | (_| |  |  Version 0.3.0-prerelease+490 (2013-12-15 07:16 UTC)
 _/ |\__'_|_|_|\__'_|  |  Commit f8f3190* (0 days old master)
|__/                   |  x86_64-linux-gnu

julia> # load the package

julia> using DataFrames

julia> # read a txt file into dataframe

julia> df1 = readtable("credit_count.txt");

julia> # subset the dataframe

julia> df2 = df1[:(CARDHLDR .== 1), ["DEFAULT", "MAJORDRG", "MINORDRG"]];

julia> # aggregate the data

julia> df3 = by(df2, "DEFAULT", :(MAJOR_DRG = mean(_DF["MAJORDRG"])))
2x2 DataFrame:
        DEFAULT MAJOR_DRG
[1,]          0  0.139851
[2,]          1  0.175703


julia> df4 = by(df2, "DEFAULT", :(MINOR_DRG = mean(_DF["MINORDRG"])))
2x2 DataFrame:
        DEFAULT MINOR_DRG
[1,]          0  0.213196
[2,]          1  0.292169


julia> # join two dataframes

julia> df5 = join(df3, df4, on = "DEFAULT", kind = :inner)
2x3 DataFrame:
        DEFAULT MAJOR_DRG MINOR_DRG
[1,]          0  0.139851  0.213196
[2,]          1  0.175703  0.292169