#collection of functions used for ICOADS analysis
 # get_icoads=function(year) : returns list of read files. For now this is first 173 characters. 
 # read.fwf2 - this reads fwf files from object previously read by readLines
 # gunzip.exe and tar.exe are needed in working directory 
 # requires format information in file info downloaded from CA

#gunzip.exe and tar.exe are needed in working directory 
    download.file("http://www.climateaudit.info/scripts/compression/gunzip.exe","gunzip.exe",mode="wb")
    download.file("http://www.climateaudit.info/scripts/compression/tar.exe","tar.exe",mode="wb")
    download.file("http://www.climateaudit.info/scripts/icoads/info_icoads.tab","info.dat",mode="wb"); load("info.dat")
###

get.icoads=function(year,Info=info, short=c("YR" ,  "MO" ,  "DY" ,  "HR"  , "LAT" , "LON",  "IM" ,  "ATTC",  "NID" , "II",   "ID",  
   	"C1" , "SLP",  "IT" ,  "AT" , "SI" ,  "SST" ,  "DCK" , "SID",  "PT")) {
	file.remove(list.files()[grep("IMMA",list.files())])
	file.remove(list.files()[grep("temp",list.files())])
	loc=paste("ftp://ftp.ncdc.noaa.gov/pub/data/icoads/IMMA.",year,".tar",sep="")
	download.file(loc,"JMMA.tar",mode="wb")
 	system(paste('"d:/climate/scripts/compression/tar.exe"','-xvf','"JMMA.tar"' )) 
	index=list.files()[grep("IMMA",list.files())];index
	M=length(index); 
	temp=rep(TRUE,12); 
	if(M<12) temp=!is.na(match(1:12,as.numeric(substr(index,11,12)) ) ) 
	widths1=c(0,cumsum(Info$width));K=length(widths1)
	widths1=cbind(widths1[1:(K-1)]+1,widths1[2:K])
	temp1= !is.na(match(Info$id,short))
	widths1= widths1[temp1,]
	sst=rep(list(NA),12)
	for(i in (1:12)[temp]) {
	  file.rename(index[i],"temp.Z")
	  system(paste('"gunzip.exe"','"temp"')) #OK
          working=readLines("temp",n=-1)
	  X=read.fwf2(working,widths=widths1,colClasses=Info$class[temp1])
	  names(X)=info$id[temp1]
	  sst[[i]]=X
	  file.remove("temp")
	}
	return(sst)
}  

#short permits temperature-relevant subset to be extracted and saved - saves some time

read.fwf2=function(working,widths,colClasses="character",short) {
	N=length(working);K=nrow(widths)
	temp1= colClasses=="numeric"
	X= array(NA,dim=c(N,K)); X=data.frame(X)
	for(k in 1:K) X[,k]=substr(working,widths[k,1],widths[k,2])
	for (k in (1:K)[temp1]) X[,k]=as.numeric(X[,k])
	 ##for (k in (1:K)[!temp1]) X[,k]=gsub(" +$","",X[,k])
	return(X)
	}


#	file.remove(list.files()[grep("IMMA",list.files())])
#	file.remove(list.files()[grep("temp",list.files())])
#	sst=get.icoads(1850)
#	sapply(sst,dim)
 	  #[1,] 4036 3364 4330 4365 4650 4824 4886 4228 3937  3609  3724  4225
	  #[2,]   20   20   20   20   20   20   20   20   20    20    20    20

#gives gridcell from ICOADS format
jones2 =function(lat,long) {
        i<-18-floor(lat/5);
        j<- 35+ceiling(long/5);
        jones2<-72*(i-1) + 1+j%%72
        jones2
        }