Edit File by line

<!doctype html>

[0] Fix | Delete

[1] Fix | Delete

<title>CodeMirror: Q mode</title>

[2] Fix | Delete

[3] Fix | Delete

[4] Fix | Delete

[5] Fix | Delete

[6] Fix | Delete

[7] Fix | Delete

[8] Fix | Delete

[9] Fix | Delete

[10] Fix | Delete

[11] Fix | Delete

<a href="http://codemirror.net"><h1>CodeMirror</h1><img id=logo src="../../doc/logo.png"></a>

[12] Fix | Delete

[13] Fix | Delete

<ul>

[14] Fix | Delete

[15] Fix | Delete

<li><a href="../../doc/manual.html">Manual</a>

[16] Fix | Delete

[17] Fix | Delete

</ul>

[18] Fix | Delete

<ul>

[19] Fix | Delete

<li><a href="../index.html">Language modes</a>

[20] Fix | Delete

[21] Fix | Delete

</ul>

[22] Fix | Delete

</div>

[23] Fix | Delete

[24] Fix | Delete

[25] Fix | Delete

[26] Fix | Delete

[27] Fix | Delete

[28] Fix | Delete

[29] Fix | Delete

/ utilities to quickly load a csv file - for more exhaustive analysis of the csv contents see csvguess.q

[30] Fix | Delete

/ 2009.09.20 - updated to match latest csvguess.q

[31] Fix | Delete

[32] Fix | Delete

/ .csv.colhdrs[file] - return a list of colhdrs from file

[33] Fix | Delete

/ info:.csv.info[file] - return a table of information about the file

[34] Fix | Delete

/ columns are:

[35] Fix | Delete

/ c - column name; ci - column index; t - load type; mw - max width;

[36] Fix | Delete

/ dchar - distinct characters in values; rule - rule that caught the type

[37] Fix | Delete

/ maybe - needs checking, _could_ be say a date, but perhaps just a float?

[38] Fix | Delete

/ .csv.info0[file;onlycols] - like .csv.info except that it only analyses <onlycols>

[39] Fix | Delete

/ example:

[40] Fix | Delete

/ info:.csv.info0[file;(.csv.colhdrs file)like"*price"]

[41] Fix | Delete

/ info:.csv.infolike[file;"*price"]

[42] Fix | Delete

/ show delete from info where t=" "

[43] Fix | Delete

/ .csv.data[file;info] - use the info from .csv.info to read the data

[44] Fix | Delete

/ .csv.data10[file;info] - like .csv.data but only returns the first 10 rows

[45] Fix | Delete

/ bulkload[file;info] - bulk loads file into table DATA (which must be already defined :: DATA:() )

[46] Fix | Delete

/ .csv.read[file]/read10[file] - for when you don't care about checking/tweaking the <info> before reading

[47] Fix | Delete

[48] Fix | Delete

\d .csv

[49] Fix | Delete

DELIM:","

[50] Fix | Delete

ZAPHDRS:0b / lowercase and remove _ from colhdrs (junk characters are always removed)

[51] Fix | Delete

WIDTHHDR:25000 / number of characters read to get the header

[52] Fix | Delete

READLINES:222 / number of lines read and used to guess the types

[53] Fix | Delete

SYMMAXWIDTH:11 / character columns narrower than this are stored as symbols

[54] Fix | Delete

SYMMAXGR:10 / max symbol granularity% before we give up and keep as a * string

[55] Fix | Delete

FORCECHARWIDTH:30 / every field (of any type) with values this wide or more is forced to character "*"

[56] Fix | Delete

DISCARDEMPTY:0b / completely ignore empty columns if true else set them to "C"

[57] Fix | Delete

CHUNKSIZE:50000000 / used in fs2 (modified .Q.fs)

[58] Fix | Delete

[59] Fix | Delete

k)nameltrim:{$[~@x;.z.s'x;~(*x)in aA:.Q.a,.Q.A;(+/&\~x in aA)_x;x]}

[60] Fix | Delete

k)fs2:{[f;s]((-7!s)>){[f;s;x]i:1+last@&0xa=r:1:(s;x;CHUNKSIZE);f@`\:i#r;x+i}[f;s]/0j}

[61] Fix | Delete

cleanhdrs:{{$[ZAPHDRS;lower x except"_";x]}x where x in DELIM,.Q.an}

[62] Fix | Delete

cancast:{nw:x$"";if[not x in"BXCS";nw:(min 0#;max 0#;::)@\:nw];$[not any nw in xjQuery(11&count y)#y;$[11<count y;not any nw in x$y;1b];0b]}

[63] Fix | Delete

[64] Fix | Delete

read:{[file]data[file;info[file]]}

[65] Fix | Delete

read10:{[file]data10[file;info[file]]}

[66] Fix | Delete

[67] Fix | Delete

colhdrs:{[file]

[68] Fix | Delete

`$nameltrim DELIM vs cleanhdrs first read0(file;0;1+first where 0xa=read1(file;0;WIDTHHDR))}

[69] Fix | Delete

data:{[file;info]

[70] Fix | Delete

(exec c from info where not t=" ")xcol(exec t from info;enlist DELIM)0:file}

[71] Fix | Delete

data10:{[file;info]

[72] Fix | Delete

data[;info](file;0;1+last 11#where 0xa=read1(file;0;15*WIDTHHDR))}

[73] Fix | Delete

info0:{[file;onlycols]

[74] Fix | Delete

colhdrs:`$nameltrim DELIM vs cleanhdrs first head:read0(file;0;1+last where 0xa=read1(file;0;WIDTHHDR));

[75] Fix | Delete

loadfmts:(count colhdrs)#"S";if[count onlycols;loadfmts[where not colhdrs in onlycols]:"C"];

[76] Fix | Delete

breaks:where 0xa=read1(file;0;floor(10+READLINES)*WIDTHHDR%count head);

[77] Fix | Delete

nas:count as:colhdrs xcol(loadfmts;enlist DELIM)0:(file;0;1+last((1+READLINES)&count breaks)#breaks);

[78] Fix | Delete

info:([]c:key flip as;v:value flip as);as:();

[79] Fix | Delete

reserved:key`.q;reserved,:.Q.res;reserved,:`i;

[80] Fix | Delete

info:update res:c in reserved from info;

[81] Fix | Delete

info:update ci:i,t:"?",ipa:0b,mdot:0,mw:0,rule:0,gr:0,ndv:0,maybe:0b,empty:0b,j10:0b,j12:0b from info;

[82] Fix | Delete

info:update ci:`s#ci from info;

[83] Fix | Delete

if[count onlycols;info:update t:" ",rule:10 from info where not c in onlycols];

[84] Fix | Delete

info:update sdv:{string(distinct x)except`}peach v from info;

[85] Fix | Delete

info:update ndv:count each sdv from info;

[86] Fix | Delete

info:update gr:floor 0.5+100*ndv%nas,mw:{max count each x}peach sdv from info where 0<ndv;

[87] Fix | Delete

info:update t:"*",rule:20 from info where mw>.csv.FORCECHARWIDTH; / long values

[88] Fix | Delete

info:update t:"C "[.csv.DISCARDEMPTY],rule:30,empty:1b from info where t="?",mw=0; / empty columns

[89] Fix | Delete

info:update dchar:{asc distinct raze x}peach sdv from info where t="?";

[90] Fix | Delete

info:update mdot:{max sum each"."=x}peach sdv from info where t="?",{"."in x}each dchar;

[91] Fix | Delete

info:update t:"n",rule:40 from info where t="?",{any x in"0123456789"}each dchar; / vaguely numeric..

[92] Fix | Delete

info:update t:"I",rule:50,ipa:1b from info where t="n",mw within 7 15,mdot=3,{all x in".0123456789"}each dchar,.csv.cancast["I"]peach sdv; / ip-address

[93] Fix | Delete

info:update t:"J",rule:60 from info where t="n",mdot=0,{all x in"+-0123456789"}each dchar,.csv.cancast["J"]peach sdv;

[94] Fix | Delete

info:update t:"I",rule:70 from info where t="J",mw<12,.csv.cancast["I"]peach sdv;

[95] Fix | Delete

info:update t:"H",rule:80 from info where t="I",mw<7,.csv.cancast["H"]peach sdv;

[96] Fix | Delete

info:update t:"F",rule:90 from info where t="n",mdot<2,mw>1,.csv.cancast["F"]peach sdv;

[97] Fix | Delete

info:update t:"E",rule:100,maybe:1b from info where t="F",mw<9;

[98] Fix | Delete

info:update t:"M",rule:110,maybe:1b from info where t in"nIHEF",mdot<2,mw within 4 7,.csv.cancast["M"]peach sdv;

[99] Fix | Delete

info:update t:"D",rule:120,maybe:1b from info where t in"nI",mdot in 0 2,mw within 6 11,.csv.cancast["D"]peach sdv;

[100] Fix | Delete

info:update t:"V",rule:130,maybe:1b from info where t="I",mw in 5 6,7<count each dchar,{all x like"*[0-9][0-5][0-9][0-5][0-9]"}peach sdv,.csv.cancast["V"]peach sdv; / 235959 12345

[101] Fix | Delete

info:update t:"U",rule:140,maybe:1b from info where t="H",mw in 3 4,7<count each dchar,{all x like"*[0-9][0-5][0-9]"}peach sdv,.csv.cancast["U"]peach sdv; /2359

[102] Fix | Delete

info:update t:"U",rule:150,maybe:0b from info where t="n",mw in 4 5,mdot=0,{all x like"*[0-9]:[0-5][0-9]"}peach sdv,.csv.cancast["U"]peach sdv;

[103] Fix | Delete

info:update t:"T",rule:160,maybe:0b from info where t="n",mw within 7 12,mdot<2,{all x like"*[0-9]:[0-5][0-9]:[0-5][0-9]*"}peach sdv,.csv.cancast["T"]peach sdv;

[104] Fix | Delete

info:update t:"V",rule:170,maybe:0b from info where t="T",mw in 7 8,mdot=0,.csv.cancast["V"]peach sdv;

[105] Fix | Delete

info:update t:"T",rule:180,maybe:1b from info where t in"EF",mw within 7 10,mdot=1,{all x like"*[0-9][0-5][0-9][0-5][0-9].*"}peach sdv,.csv.cancast["T"]peach sdv;

[106] Fix | Delete

info:update t:"Z",rule:190,maybe:0b from info where t="n",mw within 11 24,mdot<4,.csv.cancast["Z"]peach sdv;

[107] Fix | Delete

info:update t:"P",rule:200,maybe:1b from info where t="n",mw within 12 29,mdot<4,{all x like"[12]*"}peach sdv,.csv.cancast["P"]peach sdv;

[108] Fix | Delete

info:update t:"N",rule:210,maybe:1b from info where t="n",mw within 3 28,mdot=1,.csv.cancast["N"]peach sdv;

[109] Fix | Delete

info:update t:"?",rule:220,maybe:0b from info where t="n"; / reset remaining maybe numeric

[110] Fix | Delete

info:update t:"C",rule:230,maybe:0b from info where t="?",mw=1; / char

[111] Fix | Delete

info:update t:"B",rule:240,maybe:0b from info where t in"HC",mw=1,mdot=0,{$[all x in"01tTfFyYnN";(any"0fFnN"in x)and any"1tTyY"in x;0b]}each dchar; / boolean

[112] Fix | Delete

info:update t:"B",rule:250,maybe:1b from info where t in"HC",mw=1,mdot=0,{all x in"01tTfFyYnN"}each dchar; / boolean

[113] Fix | Delete

info:update t:"X",rule:260,maybe:0b from info where t="?",mw=2,{$[all x in"0123456789abcdefABCDEF";(any .Q.n in x)and any"abcdefABCDEF"in x;0b]}each dchar; /hex

[114] Fix | Delete

info:update t:"S",rule:270,maybe:1b from info where t="?",mw<.csv.SYMMAXWIDTH,mw>1,gr<.csv.SYMMAXGR; / symbols (max width permitting)

[115] Fix | Delete

info:update t:"*",rule:280,maybe:0b from info where t="?"; / the rest as strings

[116] Fix | Delete

/ flag those S/* columns which could be encoded to integers (.Q.j10/x10/j12/x12) to avoid symbols

[117] Fix | Delete

info:update j12:1b from info where t in"S*",mw<13,{all x in .Q.nA}each dchar;

[118] Fix | Delete

info:update j10:1b from info where t in"S*",mw<11,{all x in .Q.b6}each dchar;

[119] Fix | Delete

select c,ci,t,maybe,empty,res,j10,j12,ipa,mw,mdot,rule,gr,ndv,dchar from info}

[120] Fix | Delete

info:info0[;()] / by default don't restrict columns

[121] Fix | Delete

infolike:{[file;pattern] info0[file;{x where x like y}[lower colhdrs[file];pattern]]} / .csv.infolike[file;"*time"]

[122] Fix | Delete

[123] Fix | Delete

\d .

[124] Fix | Delete

/ DATA:()

[125] Fix | Delete

bulkload:{[file;info]

[126] Fix | Delete

if[not`DATA in system"v";'`DATA.not.defined];

[127] Fix | Delete

if[count DATA;'`DATA.not.empty];

[128] Fix | Delete

loadhdrs:exec c from info where not t=" ";loadfmts:exec t from info;

[129] Fix | Delete

.csv.fs2[{[file;loadhdrs;loadfmts] `DATA insert $[count DATA;flip loadhdrs!(loadfmts;.csv.DELIM)0:file;loadhdrs xcol(loadfmts;enlist .csv.DELIM)0:file]}[file;loadhdrs;loadfmts]];

[130] Fix | Delete

count DATA}

[131] Fix | Delete

@[.:;"\\l csvutil.custom.q";::]; / save your custom settings in csvutil.custom.q to override those set at the beginning of the file

[132] Fix | Delete

</textarea></div>

[133] Fix | Delete

[134] Fix | Delete

[135] Fix | Delete

var editor = CodeMirror.fromTextArea(document.getElementById("code"), {

[136] Fix | Delete

lineNumbers: true,

[137] Fix | Delete

matchBrackets: true

[138] Fix | Delete

});

[139] Fix | Delete

</script>

[140] Fix | Delete

[141] Fix | Delete

<p><strong>MIME type defined:</strong> <code>text/x-q</code>.</p>

[142] Fix | Delete

</article>

[143] Fix | Delete

[144] Fix | Delete