text:
<repo_name>ramnathv/blog<file_sep>/template/layouts/default.R

(function(page){
    links <- list(c("/index.html", "Home"),
                  c("/pages/about.html", "About"),
                  c("/pages/research.html", "Research"),
                  c("/pages/papers.html", "Papers"),
                  c("https://github.com/DASpringate", "Github"))
    blogroll <- list(c("http://www.r-bloggers.com/", "R bloggers"),
                     c("http://planet.clojure.in/", "Planet Clojure"),
                     c("http://planetbigdata.com/", "Planet Big Data"),
                     c("http://www.statsblogs.com/", "Statsblogs"))
    webdoc("html5",
           html_head(title = "<NAME> blog",
                     '<meta charset="utf-8"><meta content="width=device-width, initiali-scale=1.0, user-scalable=yes" name="viewport">',
                     link.to(url = "/rss.xml", opts = list(rel="alternate",
                                                           title="<NAME> blog RSS feed",
                                                           type="application/rss+xml")),
                     include.textfile(file.path(site, "template/resources/html/google_analytics")),
                     m("div.container-fluid well",
                       m("h1", "What is this?"),
                       m("p", "<NAME> blog on programming, data, informatics and biostatistics ")),
                     m("div.subnav",
                       unordered.list(lapply(links, function(x) link.to(x[1], x[2])),
                                      list.opts = list(class="nav nav-pills"))),
                     '<link href="/css/bootstrap.min.css" rel="stylesheet" type="text/css">',
                     '<link href="/css/bootstrap.min.css" rel="stylesheet" type="text/css">',
                     '<link href="/css/smartphone.css" media="only screen and (max-device-width:480px)" rel="stylesheet" type="text/css">'),
           html_body(m("div.container-fluid",
                       m("div.row-fluid",
                         m("div.span1"),
                         m("div.span9", content(page,
                                                include.textfile(file.path(site, "template/resources/html/disqus")))),
                         m("div.span2",
                           include.textfile(file.path(site, "template/resources/html/feedburner")),
                           m("h3", "Tags"),
                           html.taglist(site),
                           m("h3", "Blogs"),
                           unordered.list(lapply(blogroll, function(x) link.to(x[1], x[2]))))),
                       m("div.span12",
                         m("div.span2"),
                         m("div.span8",
                           m("footer.footer",
                             m("p.right back_to_top",
                               link.to("#", "⇑ Page Top")),
                             m("p",
                               link.to("http://twitter.com/datajujitsu", "@datajujitsu"),
                               "  2013 ", link.to("https://github.com/DASpringate/samatha", "Built in R with Samatha"))))))))
})(page)<file_sep>/template/posts/2013_05_16_Functional_programming_in_R.md
% R functional-programming Lisp Closures

Functional programming in R
===========================

_This post is based on a talk I gave at the [Manchester R User Group](http://www.meetup.com/Manchester-R/) on functional programming in R on May 2nd 2013.  The original slides can be found [here](http://www.slideshare.net/DASpringate/functional-programming-in-r)_

This post is about functional programming, why it is at the heart of the R language and how it can hopefully help you to write cleaner, faster and more bug-free R programs.
I will discuss what functional programming is at a very abstract level as a means of the representation of some simplified model of reality on a computer.  Then I’ll talk about the elements that functional programming is comprised of and highlight the most important elements in programming in R.  I will then go through a quick example demo of a FP-style generic bootstrap algorithm to sample linear models and return bootstrap confidence intervals.  I’ll compare this with a non-FP alternative version so you will hopefully clearly  see the advantages of using an FP style.  To wrap up, I’ll make a few suggestions for places to go to if you want to learn more about functional programming in R.

What is Functional programming?
-------------------------------

... Well, what is programming?  When you write a program you are building an  abstract representation of some tiny subset of reality on your computer, whether it is an experiment you have conducted or a model of some financial system or a collection of features of members of a population. There are obviously different ways to represent reality, and the different different methods of doing so programmatically can be thought of as the metaphysics of different styles of programming.

Consider for a moment building a representation of a river on a computer, a model of a river system for example.

![alt River](/img/river.jpg)

In non-functional languages such as C++, Java and (to some extent) Python, the river is an object in itself, a `thing` that does `things` to other `things` and that may have various properties associated with it such as flow rate, depth and pollution levels.  These properties may change over time but there is always this constant, the river, which somehow persists over time.

In FP we look at things differently...

![Hereclitus - We never step into the same river twice](/img/heraclitus.jpg)

The presocratic philosopher  Hereclitus said “We never step into the same river twice”, recognising that the thing we call a river is not really an object in itself, but something undergoing constant change through a variety of processes. In functional programming we are less concerned with the object of the river itself but rather the processes that it undergoes through time.  Our river at any point in time is just a collection of values (say, particles and their positions). These values then feed into a process to generate the series of values at the next time point.  So we have data flowing through processes of functions and that collection of data over time is what we call a river, which is really defined by the functions that the data flows through.  This is a very different way of looking at things to that of imperative, object oriented programming.

After this somewhat abstract and philosophical start, I'll talk about the more practical elements of functional programming (FP).  FP has been around for a very long time and originally  stems from Lisp, which was first implemented in the 1950’s. It is making something of a comeback of late for a variety of reasons, but mostly because it is so good at dealing with concurrent, multicore problems potentially over many computers.  There are several elements that FP is generally considered to be comprised of.  Different languages highlight different elements, depending on how strictly functional they are.

### Functions are first class citizens of the language

This means that functions can be treated just like any other data type - They can be passed around as arguments to other functions, returned by other functions and stored in lists.  This is the really big deal about functional programming and allows for higher-order functions (such as `lapply`) and closures, which I'll talk about later.  This is the most fundamental functional concept and I'd argue that a language has to have this property in order to be called a functional language, even if it has some of the other elements listed below.  For example, Python has anonymous functions and supports declarative programming with list comprehensions and generators, but functions are fundamentally different from data-types such as lists so Python cannot really be described as a functional language in the same way as Scheme or R can be.

### Functional purity

This is more of an element of good functional program design.  Pure functions take arguments, return values and otherwise have no side effects - no I/O or global variable changes.  This means that if you call a pure function twice with the same arguments, you know it will return the same value.  This means programs are easily tested because you can test different elements in isolation and once you know they work, you can treat them like a black box, knowing that they will not change some other part of your code somewhere else.  Some very strictly functional languages, like Haskell, insist on functional purity to the extent that in order to output data or read or write files you are forced to wrap your 'dirty' functions in constructs called monads to preserve the purity of your code.  R does not insist on functional purity, but it is often good practice to split your code into pure and impure functions.  This means you can test your pure code easily and confine your I/O and random numbers etc to a small number of dirty functions.

### Vectorised functions

Vectorised functions operate equally well on all elements of a vector as they do on a single number.  They are very important in R programming to the point that much of the criticism of R as a `really` slow language can be put down to failing to properly understand vectorisation.  This also includes the declarative style of programming, where you tell the language what you want, rather than how you want to get it.  This is common in languages like SQL and in Python generators.  I'll discuss this more later.

### Anonymous functions

In FP, naming and applying a function are two separate operations, you don't need to give your functions names in order to call them. So, calling this function:

```r
powfun <- function(x, pow) {
    x^pow
}
powfun(2, 10)
```

```
## [1] 1024
```

to the interpreter is exactly the same as applying variables to the anonymous function:

```r
(function(x, pow) {
    x^pow
})(2, 10)
```

```
## [1] 1024
```

This is particularly useful when you are building small, single use functions such as those used as arguments in higher order functions such as `lapply`.

### Immutable data structures

Immutable data structures are associated with pure functions. The idea is that once an object such as a vector or list is created, it should not be changed.  You can't affect your data structures via side effects from other functions. Going back to our river example, doing so would be like going back in time and rearranging some of the molecules and starting again.  Having immutable objects means that you can reason more easily about what is going on in your program. Some languages, like Clojure, only have immutable data structures and it is impossible to change a list in place, you would have to have a list as an argument to a function which returns another list that you then assign back to the variable name for the original list.  R does not insist on immutability, but in general, data structures are only changed in this way and not through side effects.  It is often best to follow this, for the same reasons as it is best to have pure functions.

### Recursion

Recursive functions are functions that call themselves.  Historically, these have been hugely important in FP, to the extent that some languages (for example Scheme) do not even have `for` loops and they define all of their looping constructs via recursion.  R does allow for recursive functions and they can sometimes be useful, particularly in traversal of tree-like data structures, but recursion in not very efficient R (it is not [tail-call optimised](http://en.wikipedia.org/wiki/Tail-call_optimization)) and I will not discuss it further here, though it may well be the subject of a future post.

Functional Programming in R
---------------------------

R has a reputation for being an ugly, hacked together and slow language. I think this is slightly unfair, but in this ever-so-slightly biassed account, I am going to blame the parents:

![R genealogy](/img/R_genealogy.png)

R is the offspring of the languages `S` and `Scheme`.  S is a statistical language invented in the 1970's which is itself based on non-functional, imperative languages like C and Fortran.  It is useful in this domain and much of R's statistical abilities stem from this, but it is certainly less than pretty.  Scheme is a concise, elegant, functional language in the lisp family. The designers of R tried to build something with the statistical functionality of S and the elegance of Scheme.  Unfortunately, they left in much of the inelegant stuff from S as well and this mixed parentage means that it is now perfectly possible to write ugly, hacky, slow code in the style of S, just as it is also possible to write elegant, efficient functional code in the style of scheme.  The problem is that functional programming has been far less mainstream so people tend to learn to code in the way they know first, resulting in rafts of ugly, hacky R code.  Programming R in an elegant, functional way is not more difficult, but is immediately less intuitive to people who were brought up reading and writing imperative code.  I would always recommend people learning R to learn these functional concepts from the outset because this way you are working with how the language was designed, rather than against it.

To show just how functional a language is at its core, it is first important to recognise that everything in R is a function call, even if it looks like it isn't. So,

```r
> 1 + 2
```

```
## [1] 3
```

... is exactly the same as...

```r
> `+`(1, 2)
```

```
## [1] 3
```

The `+` operator is really just "syntactic sugar" for a `+` function with the arguments 1 and 2 applied to it. Similarly,

```r
> 1:10
```

```
##  [1]  1  2  3  4  5  6  7  8  9 10
```

... is the same as...

```r
> `:`(1, 10)
```

```
##  [1]  1  2  3  4  5  6  7  8  9 10
```

here again, to give the range of numbers between 1 and 10, `:` is really just a function in disguise.  If you were to break down more complex expressions in this way, the result would be code that looks very Scheme-like indeed.

I will now look in more depth at the functional concepts that are most important in R, Vectorised functions, higher order functions and closures.

### Vectorised functions

Probably the best known FP concept in R is the vectorised function which 'automagically' operates on a data structure like a vector in just the same way as on a single number.  Some of the most important of these are the [vector subsetting](http://rpubs.com/daspringate/subsetting) operations. In these, you take a declarative approach to programming: you tell R what you want, not how to get it.  Because of this property of operating across vectors, proper use of vectorised functions will drastically reduce the number of loops you have to hand code into your scripts.  They are still performing loops under the hood, but these loops are implemented in C, so are many times faster than a standard for loop in R.

For example, when I was first using R for data management and analysis, I spent months writing code like this:

```r
> # Get all even numbers up to 200000
> # using S-style vector allocation:
> x <- c()
> for(i in 1:200000){
>     if(i %% 2 == 0){
>         x <- c(x, i)
>     }
> }
```

This is about the worst possible way of achieving the given task (here, getting all even numbers up to 200000).  You are running a `for` loop, which is slow in itself, and testing if `i` is even on each iteration and then growing the `x` vector every time you find that it is.  The code is ugly, slow and verbose (On my machine it took around 10 seconds).

For me, writing vectorised code was a real revelation.  To achieve the same goal as the code above in a vectorised style:

```r
> # FP style vectorised operation
> a <- 1:200000
> x <- a[a %% 2 == 0]
```

You assign a vector with all the values from 1 t0 200000, then you say "I want all of these that are divisible by two".  This ran 3 orders of magnitude faster than the non-FP code, is half the length and clearer - you don't have to mentally run through the loop in order to work out what it does.  So you get the benefits of both concision (Number of bugs correlate well with lines of code) and clarity (The code becomes almost self-documenting).

This is a slightly unfair comparison and there are ways to speed up your loops, for example by pre-allocating a vector of the correct length before you run the loop.  However, even if you do this, the result will still be around 20 times slower and will be even more verbose.  It is good practice whenever you write a `for` loop in R to check if there is not a better way to do so using vectorised functions. The majority of R's built-in functions are vectorised and using these effectively is a prerequisite of using R effectively.

### Higher-order functions

Because functions in R are first class citizens of the language, it is trivial to write and use functions that take other functions as arguments.  The most well used of these are the functions in the apply family (`lapply`, `sapply`, `apply` etc.).  These cause a lot of headaches for new-ish R users, who have just got to grips with for loops, but they are really no more difficult to use.  When you use one of these apply functions, you are just taking a collection of data (say a list or vector) and applying the input function to every element of the collection in turn, and collecting the results in another collection.

![Apply functions](/img/apply_functions.png)

Because the mapping of each element to the function is independent of the elements around it, you can split the collections up and join them together at the end, which means that the functions can be better optimised than a `for` loop (especially in the case of `lapply`) and also easily run over multiple processor cores (see `multicore::mclapply`).

Conceptually, to use these functions you just need to think about what your input is, what the function you want to apply to each element is and what data structure you want as your output:

* `lapply` : Any collection -> FUNCTION -> list
* `sapply` : Any collection -> FUNCTION -> matrix/vector
* `apply`  : Matrix/dataframe + margin -> FUNCTION -> matrix/vector
* `Reduce` : Any collection -> FUNCTION -> single element

so if you want your output in a list, use `lapply`. If you want a vector or matrix, use `sapply`.  If you want to calculate summaries of the rows or columns of a dataframe, use `apply`. If you want to condense your dataset down into a single summary number, use `Reduce`.  There are several other functions in the same family, which all follow a similar pattern.

### Closures

Closures are at the heart of all functional programming languages.  Essentially a closure is a function to which has been added data via its arguments.  The function 'closes over' the data at the time the function was created and it is possible to access it at a later time.  Compare this to the idea of an object in languages like C++ and Java, which are data with functions attached to them.

![closures](/img/closure.png)

You can use closures to build wrappers around functions with new default values and partially apply functions and even mimic Object-oriented style objects, but possibly most interestingly, you can build functions that return other functions.  This is great if you want to call a function many times on the same dataset but with different parameters, such as in maximum-likelihood optimisation problems when you are seeking to minimise some cost function and also for randomisation and bootstrapping algorithms.

To demonstrate the usefulness of this, I am now going to build a generic bootstrapping algorithm in a functional style that can be applied to any linear model.  It will demonstrate not only functions returning functions, but higher-order functions (in this case, `sapply`), anonymous functions (in the mapping function to `sapply`) and vectorised functions.  I will then compare this against a non-FP version of the algorithm and hopefully some of the advantages of writing in an FP style in R will become clear.  Here is the code. I am doing a bootstrap of a simple linear model on the classic `iris` dataset:

```r
boot.lm <- function(formula, data, ...){
  function(){
    lm(formula=formula,
       data=data[sample(nrow(data), replace=TRUE),], ...)
  }
}

iris.boot <- boot.lm(Sepal.Length ~ Petal.Length, iris)
bstrap <- sapply(X=1:1000,
                 FUN=function(x) iris.boot()$coef)
```

That is the algorithm. The boot.lm function returns a closure.  You pass it a linear model formula and a dataframe and it returns a function with no arguments that itself returns a linear model object of a bootstrapped replicate (sample with replacement) of the supplied data.  So, the iris.boot function takes the formula of Sepal.Length~Petal.Length and the iris dataset and every time you call it it gives a new bootstrap replicate of that model on that data.  You then just need to run this 1000 times and collect the coefficients, which can be done with a one-liner sapply call.  We are using sapply because we want a matrix of coefficients with one line per replicate. The `FUN` argument to `sapply` is an anonymous function that returns the coefficients of the function.  You could have equally well have written something like

```r
get.coefs <- function(x){
    iris.boot$coef
}

bstrap <- sapply(X=1:1000,
                 FUN=get.coefs)
```

... but because the function is so short, it is no less clear to include it without a name.

Once the model has run, we can use the `apply` higher-order function to summarise the rows of `bstrap` by applying the `quantile` function to give the median and 95% confidence intervals:

```r

apply(bstrap, MARGIN=1, FUN=quantile,
      probs=c(0.025, 0.5, 0.975))
```

```
##       (Intercept) Petal.Length
## 2.5%        4.162       0.3701
## 50%         4.304       0.4098
## 97.5%       4.448       0.4472
```

This is an elegant way to solve a common analysis problem in R.  If you are running a large model and you want to speed things up (and you have a few cores free!), it is a simple task and a couple of lines of code to replace the call to `sapply` to one to `multicore::mclapply` and run the model on as many processor cores as you can.

In contrast, here is a roughly equivalent non-FP style bootstrapping algorithm:

```r
boot_lm_nf <- function(d, form, iters, output, ...){
  for(i in 1:iters){
    x <- lm(formula=form,
            data=d[sample(nrow(d),
                   replace = TRUE),], ...)[[output]]
    if(i == 1){
      bootstrap <- matrix(data=NA, nrow=iters,
                    ncol=length(x),
                    dimnames=list(NULL,names(x)))
      bootstrap[i,] <- x
    } else bootstrap[i,] <- x
  }
  bootstrap
}
```

This ugly beast is full of `for`s and `if`s and braces and brackets and double brackets.  It has a load of extra boilerplate code to define the variables and fill the matrices.  Plus, it is less generic than the FP version since you can only output the attributes of the model itself, whereas previously we could apply any function we like in place of the anonymous function in the `sapply` call.  It is more than twice as verbose and impossible to multicore without a complete rewrite.  On top of all that, getting the coefficients out in a non-FP way is a tedious task:

```r
bstrap2 <- boot_lm_nf(d=iris,
            form=Sepal.Length ~ Petal.Length,
            iters=1000, output="coefficients")
CIs <- c(0.025, 0.5, 0.975)
cbind( "(Intercept)"=quantile(bstrap2[,1],probs = CIs),
      "Petal.Length"=quantile(bstrap2[,2],probs = CIs))
```

```
##       (Intercept) Petal.Length
## 2.5%        4.169       0.3699
## 50%         4.310       0.4081
## 97.5%       4.448       0.4414
```

The code duplication in the `cbind` is a pain, as is having to name the coefficients directly.  Both of these reduce the generalisability of the algorithm.

Wrapping up
-----------

I hope I have demonstrated that writing more functional R code is

* More concise (fewer lines of code)
* Often faster (Particularly with effective vectorisation)
* Clearer and less prone to bugs (because you are abstracting away a lot of the 'how to' code)
* More elegant

R is a strongly functional language to its core and if you work with this in your code, your R hacking will be more intuitive, productive and enjoyable.

Further Reading
---------------

Here are some good and accessible resources available if you want to learn more about functional programming in general and FP in R in particular:

* [Structure and interpretation of computer programs](mitpress.mit.edu/sicp) by Abelson and Sussman is the bible of FP and is written by the creators of Scheme.  This book has been used as the core of the MIT Computer Science course since the early 1990s and is still not dated.
* [<NAME>'s in progress ebook](github.com/hadley/devtools/wiki) on Github is a fantastic resource on FP in R amongst a host of other advanced R topics.
* [The R Inferno](www.burns-stat.com/pages/Tutor/R_inferno.pdf) by <NAME> is a classic free online book on R and has a great chapter on vectorisation and when it is best to apply it.
* If you are intersted in the metaphysical stuff at the start of this post, <NAME>, the inventor of the Clojure language give [this](http://www.infoq.com/presentations/Are-We-There-Yet-Rich-Hickey) great talk on the importance of FP and the failings of the traditional OOP model.  The talk was also summarised nicely in [this](http://www.flyingmachinestudios.com/programming/the-unofficial-guide-to-rich-hickeys-brain/) blog post.

<file_sep>/template/pages/index.R
layout <- "default_nocomments.R"
title <- "home"

page <- content(m("h2", "My scribblings:"),
               html.postlist(site))
 <file_sep>/template/posts/2013_05_24_RStudio_server_tips.md
% R RStudio Multicore

Develop in RStudio, run in RScript
==================================

I have been using RStudio Server for a few months now and am finding it a great tool for R development.  The web interface is superb and behaves in almost exactly the same way as the desktop version.  However, I do have one gripe which has forced me to change my working practices slightly - It is really difficult to crash out of a frozen process.  Whereas in Console R, I could just hit Control-D to get out and back to Bash, in RStudio, while you can use the escape key to terminate an operation, if you have a big process going on everything just freezes and you can't do anything.  One way to deal with this is to kill the rstudio process in another terminal, but this kills the whole session, including the editor, and you will lose anything you haven't saved in your scripts.  This problem is exacerbated when I am trying to use run parallel processes using the `Multicore` package, because it takes an age to kill all of the extra forks first.

So, now I use RStudio for development and testing and run my final analysis scripts directly using Rscript.  I have this line of code at the start of my scripts...

```r
require(multicore)
cat(sprintf("Multicore functions running on maximum %d cores",
            ifelse(length(commandArgs(trailingOnly=TRUE)),
                   cores <- commandArgs(trailingOnly=TRUE)[1],
                   cores <- 1)))
```

```
## Multicore functions running on maximum 1 cores
```

... so when I am testing in Rstudio, cores is set to 1, but when I run as an Rscript, I can specify how many cores I want to use.  I then just add `mc.cores = cores` to all of my `mclapply` calls like this:

```r
# Example processor hungry multicore operation
mats <- mclapply(1:500,
                 function(x) matrix(rnorm(x*x),
                                    ncol = x) %*% matrix(rnorm(x*x),
                                                         ncol = x),
                 mc.cores = cores)
```

The advantage of this is that, when mc.cores are set to 1, mclapply just calls lapply which is easier to crash out of (try running the above code with cores set to more than 1 to see what I mean. It will hang for a lot longer) and produces more useful error messages:

```r
# Error handling with typos:
# mcapply:
mats <- mclapply(1:500,
                 function(x) matrix(rnor(x*x),
                                    ncol = x) %*% matrix(rnorm(x*x),
                                                         ncol = x),
                 mc.cores = 2)
```

```
## Warning: all scheduled cores encountered errors in user code
```

```r
# Falls back to lapply with 1 core:
mats <- mclapply(1:500,
                 function(x) matrix(rnor(x*x),
                                    ncol = x) %*% matrix(rnorm(x*x),
                                                         ncol = x),
                 mc.cores = 1)
```

```
## Error: could not find function "rnor"
```

Running final analysis scripts has these advantages:

* You can much more easily crash out of them if there is a problem.
* You can run several scripts in parallel without taking up console space
* You can easily redirect the std error and std out from your program to a log file to keep an eye on its progress

Running multicore R scripts in the background with automatic logging
--------------------------------------------------------------------

If you have a bunch of R scripts that might each take hours to run, you don't want to clog up your RStudio console with them. This is a useful command to effectively run a big R analysis script in the background via Rscript.  It should work equally well for Linux and Mac:

```
$ Rscript --vanilla R/myscript.R 12 &> logs/myscript.log &
```

`Rscript` runs the .R file as a standalone script, without going into the R environment.  The --vanilla flag means that you run the script without calling in your .Rprofile (which is typically set up for interactive use) and without prompting you to save a workspace image. I always run the Rscript from the save level to that which is set as the project root for Rstudio to avoid any problems because of relative paths being set up wrongly. The number after the .R file to be run is the number of cores you want to run the parallel stuff on.  Other arguments you may want to pass to R would also go here. the `&>` operator redirects both the stdin and stderror to the file logs/myscript.log (I always set up a logs directory in my R projects for this purpose). The `&` at the end runs the process in the background, so you get your bash prompt back while it is running.  Then if you want to check the progress of your script, just type:

```
$ tail -f logs/myscript.log
```

And you can watch the output of your script in real time. Hit Control-C to get back to your command prompt.  You can also use the `top` command to keep an eye on your processor usage.

If you want to kill your script, either find the PID number associated with your process in `top` and do `kill PIDNUMBER` or, if you are lazy/carefree, type `killall R` to kill any running R processes.  This will not affect your rstudio instance.
<file_sep>/template/posts/2013_04_16_scraping_metadata.md
% R Python Bioinformatics

Scraping organism metadata for Treebase repositories from GOLD using Python and R
==================================================================================

I recently wanted to get hold of habitat/phenotype/sequencing metadata for the individual organisms of  an archived [Treebase](http://treebase.org/treebase-web/home.html) project.)

The [GOLD](http://www.genomesonline.org/) database holds  more than 18000 full genomes.  For many of these it provides pretty good metadata ([GOLDcards](http://genomesonline.org/cgi-bin/GOLD/bin/GOLDCards.cgi?goldstamp=Gc00536)) which are indirectly linked  to Treebase via [NCBI](http://www.ncbi.nlm.nih.gov/taxonomy) taxa [IDs](http://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=122368).

Unfortunately GOLD does not seem to have any kind of API for systematic downloads, so I hacked together a very [quick-and-dirty scraper](https://gist.github.com/2929217) in Python that reads in taxa from a Treebase repo, follows the links to each species NCBI page and downloads the linked GOLDcard, if it exists.

Here is the code. You will need the external BeautifulSoup and lxml libraries for this to work - both are fantastic. (The Treebase repo here is from Wu et al. 2009**, just change the url string for a different repo...):

Once you have downloaded all of the available files, It would be great to have your metadata in a nice flatfile with one line per taxa, right?  I did this with a little [R script](https://gist.github.com/2929366) using the rather wonderful readHTMLtable() function in the XML (install.packages('XML')) package.

The output is a semicolon separated file with taxa in the rows and the different categories of metadata in columns. The metadata is often fairly incomplete, and  there are plenty of omissions, but hopefully it will become more useful as more deposits are made to GOLD.

** <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., <NAME>., & <NAME>. 2009. A phylogeny-driven genomic encyclopaedia of Bacteria and Archaea. Nature, 462(7276): 1056-1060.

<file_sep>/template/config/tags.R
tag.layout = "default_nocomments.R"<file_sep>/template/resources/markdown/research_text.md
I am a Postdoctoral Research Associate at the University of Manchester.
I work with large-scale Electronic Medical Record Databases containing millions of electronically captured doctor-patient consultations across hundreds of GP practices in order to address important questions in public health. I am also doing methodological work looking at the validity of such databases.

I am the developer of the [ClinicalCodes.org](http://www.clinicalcodes.org) repository for lists of clinical codes to facilitate data re-use and re-analysis in observational database research. The site will be live very soon. Follow [@medcodes](http://twitter.com/medcodes) for news on this project.

My PhD was in Evolutionary biology and I am involved in research into plastic and genetic responses to environmental changes, particularly in respect to human-mediated climate change. I have also collaborated on projects in ecology and machine learning in phylogenetics.

I am also interested in using text-mining and natural language processing to perform meta-analyses of large corpora in the scientific literature. [Here](http://vimeo.com/38628200) is a short screencast describing a project involving [text-mining the ecology literature for species interaction data](https://github.com/bergmanlab/ecologIE).")

See [my publications](/pages/papers.html) for further details of my academic research<file_sep>/template/pages/pages/about.R
layout <- "default_nocomments.R"
title <- "home"

about.me <- include.markdown(file.path(site, "template/resources/markdown/about_text.md"))
about.the.blog <- include.markdown(file.path(site, "template/resources/markdown/about_blog.md"))

page <- content(
    m("h1", "About me"),
    m("div.row-fluid",
      m("div.span4",
        image.link(uri="/img/me_crop_bw.jpg", opts= list(HEIGHT = 365, WIDTH = 339))),
        m("div.span7",
          about.me)),
    m("h1", "About this blog"),
    m("div.row-fluid",
      about.the.blog))

<file_sep>/README.md
Samatha source and html for my personal site.

* template - Samatha source code to build the site
* blog - The static site itself, built using Samatha
<file_sep>/template/resources/markdown/about_blog.md
This blog is concerned with my various interests, in and outside of academia.  In particular, data programming, informatics, text-mining and biostatistics. I am interested in open source, open access and open data in academia, bio- and health informatics and distilling complex datasets from a range of sources into meaningful information.

This blog is in its third incarnation.  I first hosted it on Blogger before moving it to [Github pages](http://pages.github.com/). Upon the move, I first built the static site in Clojure using the [Misaki](http://github.com/liquidz/misaki) Static site generator, (which I [edited slightly](http://github.com/DASpringate/misaki) to meet my needs).  Recently, I have changed again to using my own [Samatha](https://github.com/DASpringate/samatha) static site engine which is written entirely in R.  I also use [Twitter Bootstrap](http://twitter.github.com/bootstrap/) as my front-end framework, because I am lazy.  Once I stop messing around with platforms, I may get around to posting more frequently.
<file_sep>/template/posts/2013_05_31_Evolutionary_biology_social_network.md
% Python R Visualisations

Mapping academic collaborations in Evolutionary Biology
=======================================================

_This post is a republication of a visualisation I did in 2011 for my (now defunct) datajujitsu.co.uk blog.  It was a naive first attempt at web-scraping from an academic publishers website.  It was done before I was aware of the problems surrounding access to, and text-mining of, online academic content hosted by publishers such as Wiley and Elsevier. Producing such a piece now (in 2013) would certainly be regarded as a political act. The text and visualisations are unchanged from the original_

Like many people, I was immensely impressed with <NAME>ler's [global map of facebook friend connections](http://paulbutler.org/archives/visualizing-facebook-friends/), a spectacular way of visualising, and humanising, a large amount of raw data.
I was further impressed to find out that he did it solely using R. I recently found Flowingdata's [tutorial](http://flowingdata.com/2011/05/11/how-to-map-connections-with-great-circles/) on creating the same effect using flight information and got to thinking about what other datasets I could apply it to.
My original plan was to build a scraper to get all of the abstracts from a particular subject from [Pubmed](http://www.ncbi.nlm.nih.gov/pubmed/) and visualise the academic collaborations between institutions for all of these abstracts. Unfortunately though, Pubmed only stores the addresses of the institutions of the corresponding author, so I decided to stick with my own subject, evolutionary biology, and get all the abstacts from the journals [Evolution](http://onlinelibrary.wiley.com/journal/10.1111/(ISSN)1558-5646) and [Evolutionary Biology](http://onlinelibrary.wiley.com/journal/10.1111/(ISSN)1420-9101) since 2009. I could then extract these using a hacked together Python script which would then feed the addresses into the [Yahoo PlaceFinder](http://developer.yahoo.com/geo/placefinder/) api to get a data set of coordinates for each cross-institution collaboration in every paper published in the journals for the last two and a half years.
I then fed this data into R, generated great circles for each of the collaborations using the geosphere package and processed it a la FlowingData to get the following global map of academic collaboration in evolutionary biology since 2009:

![Evolution social network 2009-2011](/img/evolution_social_network.png)

You can clearly see the main hubs of collaboration in Europe and the East and West coasts of the USA, with smaller hubs in Japan and South-Eastern Australia. There are further actively collaborating institutions in South America and Africa, but almost all of their collaborations are with North american and European Universities. Looking into the data itself, the median longitude for JEB institutions is firmly in Europe, while the median longitude for Evolution in the USA (This makes sense since Evolution is based in the states while JEB is a European journal, though there is no geographic imperative to publish in either).
Technical info:
I scraped the data from the Wiley website for the two journals using Python and BeautifulSoup. For the R analysis I used the modules maps, geosphere, reshape and gdata.
<file_sep>/blog/posts/2013/05/Evolutionary_biology_social_network.html
<!DOCTYPE html><html><head title="<NAME> blog"><meta charset="utf-8"><meta content="width=device-width, initiali-scale=1.0, user-scalable=yes" name="viewport"><a rel="alternate" title="<NAME>'s blog RSS feed" type="application/rss+xml" href="/rss.xml"></a><script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','//www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-40926137-1', 'github.com');
  ga('send', 'pageview');

</script><div class="container-fluid well"><h1>What is this?</h1><p><NAME>'s blog on programming, data, informatics and biostatistics </p></div><div class="subnav"><ul class="nav nav-pills"><li><a href="/index.html">Home</a></li><li><a href="/pages/about.html">About</a></li><li><a href="/pages/research.html">Research</a></li><li><a href="/pages/papers.html">Papers</a></li><li><a href="https://github.com/DASpringate">Github</a></li></ul></div><link href="/css/bootstrap.min.css" rel="stylesheet" type="text/css"><link href="/css/bootstrap.min.css" rel="stylesheet" type="text/css"><link href="/css/smartphone.css" media="only screen and (max-device-width:480px)" rel="stylesheet" type="text/css"></head><body><div class="container-fluid"><div class="row-fluid"><div class="span1"></div><div class="span9"><h1>Mapping academic collaborations in Evolutionary Biology</h1>

<p><em>This post is a republication of a visualisation I did in 2011 for my (now defunct) datajujitsu.co.uk blog.  It was a naive first attempt at web-scraping from an academic publishers website.  It was done before I was aware of the problems surrounding access to, and text-mining of, online academic content hosted by publishers such as Wiley and Elsevier. Producing such a piece now (in 2013) would certainly be regarded as a political act. The text and visualisations are unchanged from the original</em></p>

<p>Like many people, I was immensely impressed with Paul Butler's <a href="http://paulbutler.org/archives/visualizing-facebook-friends/">global map of facebook friend connections</a>, a spectacular way of visualising, and humanising, a large amount of raw data.
I was further impressed to find out that he did it solely using R. I recently found Flowingdata's <a href="http://flowingdata.com/2011/05/11/how-to-map-connections-with-great-circles/">tutorial</a> on creating the same effect using flight information and got to thinking about what other datasets I could apply it to.
My original plan was to build a scraper to get all of the abstracts from a particular subject from <a href="http://www.ncbi.nlm.nih.gov/pubmed/">Pubmed</a> and visualise the academic collaborations between institutions for all of these abstracts. Unfortunately though, Pubmed only stores the addresses of the institutions of the corresponding author, so I decided to stick with my own subject, evolutionary biology, and get all the abstacts from the journals <a href="http://onlinelibrary.wiley.com/journal/10.1111/(ISSN">Evolution</a>1558-5646) and <a href="http://onlinelibrary.wiley.com/journal/10.1111/(ISSN">Evolutionary Biology</a>1420-9101) since 2009. I could then extract these using a hacked together Python script which would then feed the addresses into the <a href="http://developer.yahoo.com/geo/placefinder/">Yahoo PlaceFinder</a> api to get a data set of coordinates for each cross-institution collaboration in every paper published in the journals for the last two and a half years.
I then fed this data into R, generated great circles for each of the collaborations using the geosphere package and processed it a la FlowingData to get the following global map of academic collaboration in evolutionary biology since 2009:</p>

<p><img src="/img/evolution_social_network.png" alt="Evolution social network 2009-2011"/></p>

<p>You can clearly see the main hubs of collaboration in Europe and the East and West coasts of the USA, with smaller hubs in Japan and South-Eastern Australia. There are further actively collaborating institutions in South America and Africa, but almost all of their collaborations are with North american and European Universities. Looking into the data itself, the median longitude for JEB institutions is firmly in Europe, while the median longitude for Evolution in the USA (This makes sense since Evolution is based in the states while JEB is a European journal, though there is no geographic imperative to publish in either).
Technical info:
I scraped the data from the Wiley website for the two journals using Python and BeautifulSoup. For the R analysis I used the modules maps, geosphere, reshape and gdata.</p>
<h6>Posted on 2013-05-31</h6><div id="disqus_thread"></div>
    <script type="text/javascript">
        /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */
        var disqus_shortname = 'daspringate'; // required: replace example with your forum shortname

        /* * * DON'T EDIT BELOW THIS LINE * * */
        (function() {
            var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
            dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
            (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
        })();
    </script>
    <noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript>
    <a href="http://disqus.com" class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a>

</div><div class="span2"><a href="http://feeds.feedburner.com/DavidSpringatesBlog" title="Subscribe to my feed" rel="alternate" type="application/rss+xml"><img src="//feedburner.google.com/fb/images/pub/feed-icon32x32.png" alt="" style="border:0"/></a><a href="http://feeds.feedburner.com/DavidSpringatesBlog" title="Subscribe" rel="alternate" type="application/rss+xml">Subscribe </a>
<h3>Tags</h3><ul><li><a href="/tags/R.html">R (5)</a></li><li><a href="/tags/tutorials.html">tutorials (1)</a></li><li><a href="/tags/subsetting.html">subsetting (1)</a></li><li><a href="/tags/vectorisation.html">vectorisation (1)</a></li><li><a href="/tags/Python.html">Python (2)</a></li><li><a href="/tags/Bioinformatics.html">Bioinformatics (1)</a></li><li><a href="/tags/functional-programming.html">functional-programming (1)</a></li><li><a href="/tags/Lisp.html">Lisp (1)</a></li><li><a href="/tags/Closures.html">Closures (1)</a></li><li><a href="/tags/RStudio.html">RStudio (1)</a></li><li><a href="/tags/Multicore.html">Multicore (1)</a></li><li><a href="/tags/Visualisations.html">Visualisations (1)</a></li></ul><h3>Blogs</h3><ul><li><a href="http://www.r-bloggers.com/">R bloggers</a></li><li><a href="http://planet.clojure.in/">Planet Clojure</a></li><li><a href="http://planetbigdata.com/">Planet Big Data</a></li><li><a href="http://www.statsblogs.com/">Statsblogs</a></li></ul></div></div><div class="span12"><div class="span2"></div><div class="span8"><footer class="footer"><p class="right back_to_top"><a href="#">⇑ Page Top</a></p><p><a href="http://twitter.com/datajujitsu">@datajujitsu</a>  2013 <a href="https://github.com/DASpringate/samatha">Built in R with Samatha</a></p></footer></div></div></div></body></html><file_sep>/template/pages/pages/research.R
layout <- "default_nocomments.R"
title <- "research"

research_text <- include.markdown(file.path(site, "template/resources/markdown/research_text.md"))

page <- content(m("h1", "My research"),
              research_text)
<file_sep>/template/resources/markdown/about_text.md
I am a biostatistician and informatician in the [Centre for Biostatistics](http://www.population-health.manchester.ac.uk/biostatistics/) and [Primary Care Research Group](http://www.medicine.manchester.ac.uk/primarycare/) at the University of Manchester.  I work with large-scale Electronic Medical Record Databases containing millions of electronically captured doctor-patient consultations across hundreds of GP practices in order to address important questions in public health.  I am also working to improve accessibility to such data.  In 2012 I completed a PhD in evolutionary biology, studying plastic and genetic responses to environmental changes. My previous work has covered a range of topics including evolutionary genetics, artificial life, machine learning, computational biology and bioinformatics.

I have been programming seriously since 2008 and am expert in R and fluent in Python and Clojure (which is rapidly becoming my favourite language of all). I also speak conversational Common Lisp, Scheme, Haskell, Processing, Bash, SQL and Stata. I have skills and experience in acquiring, processing, modelling and visualising large and complicated datasets. My hacking interests at the moment are functional programming and exploring the possibilities of using R for general-purpose scripting.  I am always interested in discussing new startup ideas in the North West.

I am a dad of two girls, an avid reader, connoisseur of lo-fi indie pop and occasional flamenco guitarist.
You can contact me on [twitter](http://twitter.com/datajujitsu) or by email at daspringate `in or near the location of`  gmail  `a very small circular shape` com
<file_sep>/template/config/config.R
####### Samatha static site generator configuration file #######

###### RSS generation ######
rss.title <- "What is this? <NAME>'s personal blog"
domain <- "http://daspringate.github.io"
rss.description <- "<NAME>'s personal blog on programming, data, informatics, biostatistics and evolution"
rss.categories <- list("R", "Clojure", "data", "informatics", "biostatistics")

###### tag setup ######

tag.layout = "default_nocomments.R"

###### blog posts ######

post.layout = "default.R"
figure.path = "img"<file_sep>/template/layouts/rss.R
rss.title <- "What is this? <NAME>'s personal blog"
rss.link <- "http://daspringate.github.com"
rss.description <- "<NAME>'s personal blog on programming, data, informatics, biostatistics and evolution"
rss.categories <- list("R", "Clojure", "data", "informatics", "biostatistics")
<file_sep>/template/pages/pages/papers.R
layout <- "default_nocomments.R"
title <- "papers"

page <- content(
    m("h1", "This page is under construction!"))
<file_sep>/template/posts/2013_05_24_RStudio_server_tips.Rmd
% R RStudio Multicore

Develop in RStudio, run in RScript
==================================

I have been using RStudio Server for a few months now and am finding it a great tool for R development.  The web interface is superb and behaves in almost exactly the same way as the desktop version.  However, I do have one gripe which has forced me to change my working practices slightly - It is really difficult to crash out of a frozen process.  Whereas in Console R, I could just hit Control-D to get out and back to Bash, in RStudio, while you can use the escape key to terminate an operation, if you have a big process going on everything just freezes and you can't do anything.  One way to deal with this is to kill the rstudio process in another terminal, but this kills the whole session, including the editor, and you will lose anything you haven't saved in your scripts.  This problem is exacerbated when I am trying to use run parallel processes using the `Multicore` package, because it takes an age to kill all of the extra forks first.

So, now I use RStudio for development and testing and run my final analysis scripts directly using Rscript.  I have this line of code at the start of my scripts...

```{r,tidy=FALSE}
require(multicore)
cat(sprintf("Multicore functions running on maximum %d cores",
            ifelse(length(commandArgs(trailingOnly=TRUE)),
                   cores <- commandArgs(trailingOnly=TRUE)[1],
                   cores <- 1)))
```

... so when I am testing in Rstudio, cores is set to 1, but when I run as an Rscript, I can specify how many cores I want to use.  I then just add `mc.cores = cores` to all of my `mclapply` calls like this:

```{r,tidy=FALSE}
# Example processor hungry multicore operation
mats <- mclapply(1:500,
                 function(x) matrix(rnorm(x*x),
                                    ncol = x) %*% matrix(rnorm(x*x),
                                                         ncol = x),
                 mc.cores = cores)
```

The advantage of this is that, when mc.cores are set to 1, mclapply just calls lapply which is easier to crash out of (try running the above code with cores set to more than 1 to see what I mean. It will hang for a lot longer) and produces more useful error messages:

```{r,tidy=FALSE}
# Error handling with typos:
# mcapply:
mats <- mclapply(1:500,
                 function(x) matrix(rnor(x*x),
                                    ncol = x) %*% matrix(rnorm(x*x),
                                                         ncol = x),
                 mc.cores = 2)
# Falls back to lapply with 1 core:
mats <- mclapply(1:500,
                 function(x) matrix(rnor(x*x),
                                    ncol = x) %*% matrix(rnorm(x*x),
                                                         ncol = x),
                 mc.cores = 1)
```

Running final analysis scripts has these advantages:

* You can much more easily crash out of them if there is a problem.
* You can run several scripts in parallel without taking up console space
* You can easily redirect the std error and std out from your program to a log file to keep an eye on its progress

Running multicore R scripts in the background with automatic logging
--------------------------------------------------------------------

If you have a bunch of R scripts that might each take hours to run, you don't want to clog up your RStudio console with them. This is a useful command to effectively run a big R analysis script in the background via Rscript.  It should work equally well for Linux and Mac:

```
$ Rscript --vanilla R/myscript.R 12 &> logs/myscript.log &
```

`Rscript` runs the .R file as a standalone script, without going into the R environment.  The --vanilla flag means that you run the script without calling in your .Rprofile (which is typically set up for interactive use) and without prompting you to save a workspace image. I always run the Rscript from the save level to that which is set as the project root for Rstudio to avoid any problems because of relative paths being set up wrongly. The number after the .R file to be run is the number of cores you want to run the parallel stuff on.  Other arguments you may want to pass to R would also go here. the `&>` operator redirects both the stdin and stderror to the file logs/myscript.log (I always set up a logs directory in my R projects for this purpose). The `&` at the end runs the process in the background, so you get your bash prompt back while it is running.  Then if you want to check the progress of your script, just type:

```
$ tail -f logs/myscript.log
```

And you can watch the output of your script in real time. Hit Control-C to get back to your command prompt.  You can also use the `top` command to keep an eye on your processor usage.

If you want to kill your script, either find the PID number associated with your process in `top` and do `kill PIDNUMBER` or, if you are lazy/carefree, type `killall R` to kill any running R processes.  This will not affect your rstudio instance.

languages: RMarkdown Markdown R HTML

repo_language: RMarkdown

repo_name: ramnathv/blog


text:
<file_sep># GOW
God of War images with parallax effects

https://wellyngtonmolinari.github.io/GOW/

branch_name: refs/heads/main

repo_language: Markdown

repo_name: WellyngtonMolinari/GOW


text:
<repo_name>Greh/matasano<file_sep>/oneone.go
package main

import (
	"fmt"
	"os"
)

func main() {
	a := os.Args[1]
	number := []byte(a)
	fmt.Println(number)

	hexslice := toHex(number)
	fmt.Println(hexslice)
	bitslice := toBits(hexslice)
	fmt.Println(bitslice)
	b64slice := toB64(bitslice)
	fmt.Println(b64slice)
	strslice := toString(b64slice)
	fmt.Println(strslice)
}

func toHex(b []byte) []byte {
	hexslice := make([]byte, len(b), len(b))
	for i := range b {
		if (b[i] >= 48) && (b[i] <= 58) {
			hexslice[i] = b[i] - 48
		} else if (b[i] >= 97) && (b[i] <= 102) {
			hexslice[i] = b[i] - 97 + 10
		} else {
			fmt.Println("invalid input")
			return nil
		}
	}
	return hexslice
}

func toBits(h []byte) []byte {
	somebits := make([]byte, len(h)*4, len(h)*4)
	for i := range h {
		if h[i]&8 == 8 {
			somebits[i*4+0] = 1
		} else {
			somebits[i*4+0] = 0
		}
		if h[i]&4 == 4 {
			somebits[i*4+1] = 1
		} else {
			somebits[i*4+1] = 0
		}
		if h[i]&2 == 2 {
			somebits[i*4+2] = 1
		} else {
			somebits[i*4+2] = 0
		}
		if h[i]&1 == 1 {
			somebits[i*4+3] = 1
		} else {
			somebits[i*4+3] = 0
		}
	}
	return somebits
}

func toB64(b []byte) []byte {
	b64 := make([]byte, len(b)/6, len(b)/6)
	for i := range b64 {
		b64[i] = b[i*6]*32 + b[(i*6)+1]*16 + b[(i*6)+2]*8 + b[(i*6)+3]*4 + b[(i*6)+4]*2 + b[(i*6)+5]
	}

	return b64
}

func toString(b64 []byte) string {
	strslice := make([]byte, len(b64), len(b64))
	for i := range b64 {
		if b64[i] < 26 {
			strslice[i] = b64[i] + 65
		} else if b64[i] < 52 {
			strslice[i] = b64[i] + 71 //97 - 26
		} else if b64[i] < 62 {
			strslice[i] = b64[i] - 4 //-52 + 48
		} else if b64[i] == 62 {
			strslice[i] = 43
		} else if b64[i] == 63 {
			strslice[i] = 47
		} else {
			strslice[i] = 63
		}
	}
	fmt.Println("string of b64")
	strstr := string(strslice)
	return strstr
}

branch_name: refs/heads/master

repo_language: Go

repo_name: Greh/matasano


text:
<repo_name>HarikaSantapuri/A-star-search-RoutingAlgorithm<file_sep>/README.md
# A-star-search-RoutingAlgorithm
This code implements a route planning algorithm to find the shortest path between two points on a map


text:
<file_sep># softfound
software foundation readings and practices

repo_name: turfnima/softfound