From a0b48c44f9bb28423a3cf15dd1d942f6470a21a7 Mon Sep 17 00:00:00 2001 From: "R. S. Doiel" Date: Mon, 6 Feb 2017 12:40:53 -0800 Subject: [PATCH] Initial Setup --- .gitignore | 34 ++++ INSTALL.md | 80 +++++++++ LICENSE | 13 ++ Makefile | 135 +++++++++++++++ README.md | 31 ++++ TODO.md | 12 ++ assets/liblogo.gif | Bin 0 -> 4431 bytes assets/orangebooks-logo-icon.png | Bin 0 -> 1516 bytes assets/orangebooks-logo.png | Bin 0 -> 10263 bytes cmds/csv2mdtable/csv2mdtable.go | 140 +++++++++++++++ cmds/csv2xlsx/csv2xlsx.go | 184 ++++++++++++++++++++ cmds/csvcols/csvcols.go | 221 ++++++++++++++++++++++++ cmds/csvjoin/csvjoin.go | 211 +++++++++++++++++++++++ cmds/jsoncols/jsoncols.go | 199 ++++++++++++++++++++++ cmds/jsonrange/jsonrange.go | 284 +++++++++++++++++++++++++++++++ cmds/xlsx2csv/xlsx2csv.go | 219 ++++++++++++++++++++++++ cmds/xlsx2json/xlsx2json.go | 219 ++++++++++++++++++++++++ css/site.css | 245 ++++++++++++++++++++++++++ csv2mdtable.md | 37 ++++ csv2xlsx.md | 41 +++++ csvcols.md | 48 ++++++ csvjoin.md | 32 ++++ datatools.go | 28 +++ demo/demo1.json | 1 + demo/demo2.json | 1 + demo/file1.csv | 7 + demo/file2.csv | 7 + demo/schedule.xlsx | Bin 0 -> 5986 bytes demo/workbook.xlsx | Bin 0 -> 3665 bytes jsoncols.md | 69 ++++++++ jsonrange.md | 85 +++++++++ mk-website.bash | 53 ++++++ nav.md | 14 ++ page.tmpl | 29 ++++ publish.bash | 30 ++++ xlsx2csv.md | 47 +++++ xlsx2json.md | 48 ++++++ 37 files changed, 2804 insertions(+) create mode 100644 .gitignore create mode 100644 INSTALL.md create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 TODO.md create mode 100644 assets/liblogo.gif create mode 100644 assets/orangebooks-logo-icon.png create mode 100644 assets/orangebooks-logo.png create mode 100644 cmds/csv2mdtable/csv2mdtable.go create mode 100644 cmds/csv2xlsx/csv2xlsx.go create mode 100644 cmds/csvcols/csvcols.go create mode 100644 cmds/csvjoin/csvjoin.go create mode 100644 cmds/jsoncols/jsoncols.go create mode 100644 cmds/jsonrange/jsonrange.go create mode 100644 cmds/xlsx2csv/xlsx2csv.go create mode 100644 cmds/xlsx2json/xlsx2json.go create mode 100644 css/site.css create mode 100644 csv2mdtable.md create mode 100644 csv2xlsx.md create mode 100644 csvcols.md create mode 100644 csvjoin.md create mode 100644 datatools.go create mode 100644 demo/demo1.json create mode 100644 demo/demo2.json create mode 100644 demo/file1.csv create mode 100644 demo/file2.csv create mode 100644 demo/schedule.xlsx create mode 100644 demo/workbook.xlsx create mode 100644 jsoncols.md create mode 100644 jsonrange.md create mode 100755 mk-website.bash create mode 100644 nav.md create mode 100644 page.tmpl create mode 100755 publish.bash create mode 100644 xlsx2csv.md create mode 100644 xlsx2json.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..34e59d5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,34 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + +# +# Project specific files +# +*~ +*.swp +*.zip +dist/* +bin/* + diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..3734536 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,80 @@ + +# Installation + +## Compiled version + +*datatools* is a collection of command line programs run from a shell like Bash (or Powershell in Windows). If you download the repository a compiled version is in the dist directory. The compiled VERSION_NO matching your computer type and operating system can be copied to a bin directory in your PATH. + +Compiled versions are available for Mac OS X (amd64 processor), Linux (amd64), Windows (amd64) and Rapsberry Pi (both ARM6 and ARM7) + +### Mac OS X + +1. Download **datatools-VERSION_NO-release.zip** from [https://github.com/caltechlibrary/datatools/releases/latest](https://github.com/caltechlibrary/datatools/releases/latest) +2. Open a finder window, find and unzip **datatools-VERSION_NO-release.zip** +3. Look in the unziped folder and find *dist/macosx-amd64/* +4. Drag (or copy) *jsoncols*, *jsonrange*, etc. to a "bin" directory in your path +5. Open and "Terminal" and run `jsoncols -h` to confirm you were successful + +### Windows + +1. Download **datatools-VERSION_NO-release.zip** from [https://github.com/caltechlibrary/datatools/releases/latest](https://github.com/caltechlibrary/datatools/releases/latest) +2. Open the file manager find and unzip **datatools-VERSION_NO-release.zip** +3. Look in the unziped folder and find *dist/windows-amd64/* +4. Drag (or copy) *jsoncols.exe*, *jsonrange.exe*, etc. to a "bin" directory in your path +5. Open Bash and and run `jsoncols -h` to confirm you were successful + +### Linux + +1. Download **datatools-VERSION_NO-release.zip** from [https://github.com/caltechlibrary/datatools/releases/latest](https://github.com/caltechlibrary/datatools/releases/latest) +2. Find and unzip **datatools-VERSION_NO-release.zip** +3. In the unziped directory and find for *dist/linux-amd64/* +4. Copy *jsoncols*, *jsonrange*, etc. to a "bin" directory (e.g. cp ~/Downloads/datatools-VERSION_NO-release/dist/linux-amd64/\* ~/bin/) +5. From the shell prompt run `jsoncols -h` to confirm you were successful + +### Raspberry Pi + +If you are using a Raspberry Pi 2 or later use the ARM7 VERSION_NO, ARM6 is only for the first generaiton Raspberry Pi. + +1. Download **datatools-VERSION_NO-release.zip** from [https://github.com/caltechlibrary/datatools/releases/latest](https://github.com/caltechlibrary/datatools/releases/latest) +2. Find and unzip **datatools-VERSION_NO-release.zip** +3. In the unziped directory and find for *dist/raspberrypi-arm7/* +4. Copy *jsoncols*, *jsonrange*, etc. to a "bin" directory (e.g. cp ~/Downloads/datatools-VERSION_NO-release/dist/raspberrypi-arm7/\* ~/bin/) + + if you are using an original Raspberry Pi you should copy the ARM6 version instead +5. From the shell prompt run `jsoncols -h` to confirm you were successful + + +## Compiling from source + +If you have go v1.7.4 or better installed then should be able to "go get" to install all the **datatools** utilities and + +``` + go get -u github.com/caltechlibrary/datatools/... +``` + +Or for Windows 10 Powershell (assumes the Windows versions of Go and Git are previously installed) + + +```powershell + $Env:GOPATH = "$HOME" + go get -u github.com/caltechlibrary/datatools/... +``` + +or to install from source on Linux/Unix systems + +```bash + git clone https://github.com/caltechlibrary/datatools src/github.com/caltechlibrary/datatools + cd src/github.com/caltechlibrary/datatools + make + make test + make install +``` + +Or for Windows 10 Powershell + +```powershell + $Env:GOBIN = "$HOME\bin" + git clone https://github.com/caltechlibrary/datatools src/github.com/caltechlibrary/datatools + cd src\github.com\caltechlibrary\datatools + go install cmds\jsoncols\filefile.go +``` + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d910106 --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ + +Copyright (c) 2017, Caltech +All rights not granted herein are expressly reserved by Caltech. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1651c93 --- /dev/null +++ b/Makefile @@ -0,0 +1,135 @@ +# +# Simple Makefile +# +PROJECT = datatools + +VERSION = $(shell grep -m1 'Version = ' $(PROJECT).go | cut -d\" -f 2) + +BRANCH = $(shell git branch | grep '* ' | cut -d\ -f 2) + +build: bin/csvcols bin/csvjoin bin/jsoncols bin/jsonrange bin/xlsx2json bin/xlsx2csv bin/csv2mdtable bin/csv2xlsx + +bin/csvcols: datatools.go cmds/csvcols/csvcols.go + go build -o bin/csvcols cmds/csvcols/csvcols.go + +bin/csvjoin: datatools.go cmds/csvjoin/csvjoin.go + go build -o bin/csvjoin cmds/csvjoin/csvjoin.go + +bin/jsoncols: datatools.go cmds/jsoncols/jsoncols.go + go build -o bin/jsoncols cmds/jsoncols/jsoncols.go + +bin/jsonrange: datatools.go cmds/jsonrange/jsonrange.go + go build -o bin/jsonrange cmds/jsonrange/jsonrange.go + +bin/xlsx2json: datatools.go cmds/xlsx2json/xlsx2json.go + go build -o bin/xlsx2json cmds/xlsx2json/xlsx2json.go + +bin/xlsx2csv: datatools.go cmds/xlsx2csv/xlsx2csv.go + go build -o bin/xlsx2csv cmds/xlsx2csv/xlsx2csv.go + +bin/csv2mdtable: datatools.go cmds/csv2mdtable/csv2mdtable.go + go build -o bin/csv2mdtable cmds/csv2mdtable/csv2mdtable.go + +bin/csv2xlsx: datatools.go cmds/csv2xlsx/csv2xlsx.go + go build -o bin/csv2xlsx cmds/csv2xlsx/csv2xlsx.go + +website: + ./mk-website.bash + +status: + git status + +save: + git commit -am "Quick Save" + git push origin $(BRANCH) + +refresh: + git fetch origin + git pull origin $(BRANCH) + +publish: + ./mk-website.bash + ./publish.bash + +clean: + if [ -f index.html ]; then /bin/rm *.html;fi + if [ -d bin ]; then /bin/rm -fR bin; fi + if [ -d dist ]; then /bin/rm -fR dist; fi + if [ -f $(PROJECT)-$(VERSION)-release.zip ]; then rm -f $(PROJECT)-$(VERSION)-release.zip; fi + +install: + env GOBIN=$(HOME)/bin go install cmds/csvcols/csvcols.go + env GOBIN=$(HOME)/bin go install cmds/csvjoin/csvjoin.go + env GOBIN=$(HOME)/bin go install cmds/jsoncols/jsoncols.go + env GOBIN=$(HOME)/bin go install cmds/jsonrange/jsonrange.go + env GOBIN=$(HOME)/bin go install cmds/xlsx2json/xlsx2json.go + env GOBIN=$(HOME)/bin go install cmds/xlsx2csv/xlsx2csv.go + env GOBIN=$(HOME)/bin go install cmds/csv2mdtable/csv2mdtable.go + env GOBIN=$(HOME)/bin go install cmds/csv2xlsx/csv2xlsx.go + +dist/linux-amd64: + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dist/linux-amd64/csvcols cmds/csvcols/csvcols.go + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dist/linux-amd64/csvjoin cmds/csvjoin/csvjoin.go + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dist/linux-amd64/jsoncols cmds/jsoncols/jsoncols.go + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dist/linux-amd64/jsonrange cmds/jsonrange/jsonrange.go + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dist/linux-amd64/xlsx2json cmds/xlsx2json/xlsx2json.go + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dist/linux-amd64/xlsx2csv cmds/xlsx2csv/xlsx2csv.go + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dist/linux-amd64/csv2mdtable cmds/csv2mdtable/csv2mdtable.go + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dist/linux-amd64/csv2xlsx cmds/csv2xlsx/csv2xlsx.go + +dist/macosx-amd64: + env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o dist/macosx-amd64/csvcols cmds/csvcols/csvcols.go + env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o dist/macosx-amd64/csvjoin cmds/csvjoin/csvjoin.go + env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o dist/macosx-amd64/jsoncols cmds/jsoncols/jsoncols.go + env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o dist/macosx-amd64/jsonrange cmds/jsonrange/jsonrange.go + env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o dist/macosx-amd64/xlsx2json cmds/xlsx2json/xlsx2json.go + env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o dist/macosx-amd64/xlsx2csv cmds/xlsx2csv/xlsx2csv.go + env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o dist/macosx-amd64/csv2mdtable cmds/csv2mdtable/csv2mdtable.go + env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o dist/macosx-amd64/csv2xlsx cmds/csv2xlsx/csv2xlsx.go + +dist/windows-amd64: + env CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o dist/windows-amd64/csvcols.exe cmds/csvcols/csvcols.go + env CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o dist/windows-amd64/csvjoin.exe cmds/csvjoin/csvjoin.go + env CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o dist/windows-amd64/jsoncols.exe cmds/jsoncols/jsoncols.go + env CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o dist/windows-amd64/jsonrange.exe cmds/jsonrange/jsonrange.go + env CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o dist/windows-amd64/xlsx2json.exe cmds/xlsx2json/xlsx2json.go + env CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o dist/windows-amd64/xlsx2csv.exe cmds/xlsx2csv/xlsx2csv.go + env CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o dist/windows-amd64/csv2mdtable.exe cmds/csv2mdtable/csv2mdtable.go + env CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o dist/windows-amd64/csv2xlsx.exe cmds/csv2xlsx/csv2xlsx.go + +dist/raspbian-arm7: + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -o dist/raspbian-arm7/csvcols cmds/csvcols/csvcols.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -o dist/raspbian-arm7/csvjoin cmds/csvjoin/csvjoin.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -o dist/raspbian-arm7/jsoncols cmds/jsoncols/jsoncols.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -o dist/raspbian-arm7/jsonrange cmds/jsonrange/jsonrange.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -o dist/raspbian-arm7/xlsx2json cmds/xlsx2json/xlsx2json.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -o dist/raspbian-arm7/xlsx2csv cmds/xlsx2csv/xlsx2csv.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -o dist/raspbian-arm7/csv2mdtable cmds/csv2mdtable/csv2mdtable.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -o dist/raspbian-arm7/csv2xlsx cmds/csv2xlsx/csv2xlsx.go + +dist/raspbian-arm6: + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -o dist/raspbian-arm6/csvcols cmds/csvcols/csvcols.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -o dist/raspbian-arm6/csvjoin cmds/csvjoin/csvjoin.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -o dist/raspbian-arm6/jsoncols cmds/jsoncols/jsoncols.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -o dist/raspbian-arm6/jsonrange cmds/jsonrange/jsonrange.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -o dist/raspbian-arm6/xlsx2json cmds/xlsx2json/xlsx2json.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -o dist/raspbian-arm6/xlsx2csv cmds/xlsx2csv/xlsx2csv.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -o dist/raspbian-arm6/csv2mdtable cmds/csv2mdtable/csv2mdtable.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -o dist/raspbian-arm6/csv2xlsx cmds/csv2xlsx/csv2xlsx.go + + +release: dist/linux-amd64 dist/macosx-amd64 dist/windows-amd64 dist/raspbian-arm7 dist/raspbian-arm6 + mkdir -p dist + cp -v README.md dist/ + cp -v LICENSE dist/ + cp -v INSTALL.md dist/ + cp -v csvcols.md dist/ + cp -v csvjoin.md dist/ + cp -v jsoncols.md dist/ + cp -v jsonrange.md dist/ + cp -v xlsx2json.md dist/ + cp -v xlsx2csv.md dist/ + cp -v csv2mdtable.md dist/ + cp -v csv2xlsx.md dist/ + zip -r $(PROJECT)-$(VERSION)-release.zip dist/ + diff --git a/README.md b/README.md new file mode 100644 index 0000000..765f58a --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ + +# datatools + +Various utilities for simplifying JSON, Excel Workbook and CSV data work on the command line. + ++ [csvcols](csvcols.html) - a tool for formatting command line arguments intoa CSV row or filtering CSV rows for specific columns ++ [csvjoin](csvjoin.html) - a tool to join to CSV files on common values in designated columns, writes combined CSV rows to stdout ++ [csv2mdtable](csv2mdtable.html) - a tool to render CSV as a Github Flavored Markdown table ++ [jsoncols](jsoncols.html) - a tool for exploring and extracting JSON values into columns ++ [jsonrange](jsonrange.html) - a tool for iterating for JSON maps and arrays ++ [xlsx2json](xlsx2json.html) - a tool for converting Excel Workbooks to JSON files ++ [xlsx2csv](xlsx2csv.html) - a tool for converting Excel Workbooks sheets to a CSV file(s) + +Compiled versions are provided for Linux (amd64), Mac OS X (amd64), +Windows 10 (amd64) and Raspbian (ARM6, ARM7). See https://github.com/caltechlibrary/datatools. + +Use the utilities try "-help" option for a full list of options. + + +## Installation + +_datatools_ is go get-able. + +``` + go get github.com/caltechlibrary/datatools/... +``` + +Or see [INSTALL.md](install.html) for details for installing +compiled versions of the programs. + + diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..bef8ea5 --- /dev/null +++ b/TODO.md @@ -0,0 +1,12 @@ + +# Someday, Maybe + ++ csv2json would convert CSV content to a 2d-JSON array + + E.g. `cat file1.csv | csv2json` ++ csv2xlsx would take CSV content piped from stdin and write it do a sheet in an Excel file (creating the Excel workbook if needed) + + E.g. `cat file1.csv | csv2xlsx MyWorkbook.xlsx "Sheet from file1.csv"` ++ csvfind would accept CSV input from stdin and output rows with matching column values + + E.g. `cat file1.csv | csvfind -column=3 "Book"` ++ json2csv would convert a 2d JSON array to CSV output, it would comvert a JSON object/map to a column of keys next to a column of values + + E.g. `cat data.json | json2csv` + diff --git a/assets/liblogo.gif b/assets/liblogo.gif new file mode 100644 index 0000000000000000000000000000000000000000..cbd9ea0eee23dc6e0055691e551f380896260690 GIT binary patch literal 4431 zcmV-V5wPw@Nk%w1VSWN@0J9AM|NsC0|IGi)%>T@p|CyQpnP&f)Gyi5Y%*@Q0nVDv0 zW-~K000030|Nj6000000A^8LW000dDEC2ui0Db~%000F4u*pfQy*TU5yZ>M)4uWW& zXsWJk>%MR-&vb2zfG`Y2@B2Fe;70r$j>u#ZH8_-t&}c`XtRkh@tai)o5(7Mgb$sjz zy*IM*)O4cXGphOuuiNkV{0YFD`^!&yDt`wiZEk{zQ-q3*j*pNr0R@DW2L=O?Mt>to zftrqTexRhKrjV1Cs+XCk7K|W=N1v{EqI0#my1P)atCj@;yRaUTYRJsZ&LIMn zzthLA#2m)e&s)l5+uz{N+0>Q_;|rCQT&~l2&StlfCyPb~;8y{sC$=V8+_S8O!k$KB2rawzsmw@Lrii8r zwWH2zKDVW9yLhD1s&uPReMGq;;ADiWC2qXBBHRbYx?s+_5p=26>-r2&{tfa<-CN++ z``dK|*hr*jrCy!`f&~l^)SrRfw+jG?`Qe=ZAm3RQ9(|4A$H0OMH0a=i5H2WT9tbEP zp?>?t5TIi;WR~7gI)L><4_}?JLV;-2ki!lHj95Sr>}e4|h%N>pp9Batu%d`v1;K@a z5(X*ckU`!7fQ3O~nBjK7sA!1@cELBsi!p|Pq7ozspyifa$`>Ut1Nc`17*eh_K$RR& z_vIfiig^Q&M8-K}9Y+$WzWz-KXKlA-9K=pd>X znl&V#z@?YMIV7f>CSV?tcH-&6hH88WLpBA><>p$ACTFF46tr4uRkF_UXbc08I?b&c zM3zcD1403+kP2jhz_A+)8>*-)Ft@1x2XK|4stEw-xfBw?Y%&T4wNA+aN=p&2stE$< z3T&FNFlQ^JL?jUBokS?>DF$+0$m|N48Bm0x<4$4jQkNBqUa!|CVZgIGT6^tjYcYad zU&g*0g|Z9IYk{Tp!h5d>t!kHux;u3sFd?Q&(Csga{#x)62WPO=!-TL~0|F8k;VzL2 zXi(>}8=qha!bE^vPg5L>%q9llt{fY=N~d^%&^nb&1I;|&K(W0VP%Wg!JwyAXBGEQ& zh13!l&D8}-i#-dJYOm%d)H(%QL(N2Jnj{AlbKNz%BsX&MMShuWt66O;K{TX=WBYXC zYREhT-B!z~Hw8D}y>|$i$qfhp%ovd^_^cHu%sAN;q+Pgka7Qf#tIr-(> z?y1PXQ)bEK04=W4@8^yJ#<dW`=ic^q$N}4lIG^`XyOP>53&0B|h&Rlt7t6B!}QCx|i~?gQZi9R>zi6<}PC0{L3VDloAs z?imV4_8S$* zi3Eup>QI1v6U0_dc!9A0&0$(RLs3&;*3`Ll~lw2sdVKsp=U50-UJVB#Yl|RjtQ4K6y<0{&yNFPpdPB`2e9W`LbJXocOd*o0iaZx)> z7VQroq?RKsz#K-th76b>rK>{FC{wN=j*Xb(Bq>lyf@tY0Um20dUggPG@X=&?B-kHZ zc}5Y&s+J%~Vn(LI9i}PqkwU7XBxhO2EXujB4kO;1|ZWJGwPix(h|lDWRsW~@T2fXi2~$gOPFRfL76bg0(mBa zp47ajE@spRa`5y2mkkhT0{`GnYz(s_l|biO(5V_eEfktX>H-21U_n_@&{CITX$t_5 z0EjhPFZjR6%rFo70@6B{PYE1n zS2;yKjR#d)VpOB)kSZC0`O$U|VwafHstveGgbD@es~fQ2ed-mY`GsI-45FNzY*C!gxg4S-XzEUQ2}rLdJn0mZ5+pzO>j zhFJ{^31gi9eXa*UE1|Pqt-2W4Y-eGs)(&VlNb~w`sAdaM@-m=R0QEve3#VId@-{fd z1Fn0^Hl8>T!nl&18x@N{)7CaO1JQjDXa9SF>oPzB1`h4Ts!P@hB*3%;u7G$it0VI2 z2c|?F>!nszp{rpkGmzCHV0F@7Z*&vB8z5}cP_q@}vH*`Bg(n>MdjshaDZ&;QD{1>X z)(lKInsX%*X&V44w_+g3XDx8O+-hDi1=+P8n5&Xh<6M0AmW%(M&jWUJ; zj8z!s$nOGP)l7c*Y(`e=yl1NK1U87fI z9}79k*cEOi(z^#jbB?K5-ffprOi4yJ`ej%cb7IS^UsOXO%?!Zt1*h2o2xkqp5k{5; zG}W&le=ETnqu}z=>H$z`jEKF+NJrN_&zgN*IZa0=1No;?} zo6PhswyCjz>CEmsb|pV@)$lfO3AfzODHj2qYujvpd3+!qB0L4~-%*yn0)}T=Y5C(IWNT_woy7YZCK4vtJg+|%-GQv*hq#yTmD6BryvA6xz zm!17@NgL(0=epQa@On7E8HuV0@l`9KFVJt?XUd;IKNt7tEC8Pcr9pec@ILSV(~CU( zsuy+GXLGRjI0mF~Wz- z=Y}#+X(&K7kVlA#ID=H@h=UgbjF@c4;)oE)fPqM7rS^i=vkC{b8;fL#DcC2TScK`; zd2-il%@_+-&_07kEeoX=teAGL=svz^0!T88l1MLJcxS*!j*6&@aMECjw}6ni0%iD! zmB?}XM2f%Q6r7k<(`Yn|)I|0;C3FD{gjQS1XmNSibkz5apCuvVLW=-ajvAmT4=97g z7>jU1Y7)?n+V_qsfN!N^i*sih9rZsavl8TF1tUq1NN6;eM0skEURS0B$GC7Lvs2(! zkcifa-3SA)2#(`7ZR2 zipOV!95@H=l!hYZLOvuWF6k(b0YIL#jWoH8me!LCxgh!Gkmv}KwRnCHnU33|DO{tG z;un=K5RYNU0FalJ3S=T{giRI|cXMD$)v^kQR!(v_CH^Bc!PhJ9HwGfu0GFwAGJ}?3 zrj{r)eQgPdCjcsgDVIVCkx1i&XvcL98vEaw=Bvqv>jIYb)weiX2o*s?reFb$6x z5s79iy0Zg50FA)lb6aAZgNgr>n@7Md_CR9nmXo`dDE|pup){h2 zqlVNeqSncl1L~IU$e?w}g99j|AaIcjYKS1Pc!Ecv6VQ|{fNw}LkA%r+Y0;k`dWVq7 zjS*)7yxBB<@tjb~P5>%tJ4u5u3VwDumo_@12zmihmZPYZpy@e`K8j$o8H|$iV(q3m zQkppSNfY$fq$tWbk3nv$RHDxqSy$?mS*kNbvOL@8oo70Uvsa@!>UwEfq2IZY<=F!M zwTANbLr97dauY-c@gXs(Hk(nY<&cS78K8ZdgRzzWOcIHxmDp!78kOh-p@b(StMX*? z2&6$;s(bLB)XGUDq-a}qp7-bDp#ltAfY$9 zeb?1p6^5ubYbz_{OF$4;iTtl$c1w9y+VVibTQ3jT zvu$Ob0Txr4LM-spwWZjR3xKZ^u(*ke0jh?P6&AK``?g=|0jgC5U@&51 zIa8eTA{KzIN8lz)#|NZ)1nM^%MmH!j5CBhFZbR|0OzNV!2O)U%yTA*)COf>7Yo5Lv zybm&DTGL?ArDHexAi>L54Prsp)nF;xtOEk zC4w31+rH2M7FMu!e-gNB1iqHAc4t!j1c*s8G86(}(7x)s8~_}^M=?};`LmdU0Cp<_ z2yDGRhABGr23oVM+X26z#CAbroFn2RLa`$*k}5&4z$)OuHPXNJbiyETGoK3uIdDt3 zv?VAU!!q0t3PQOve8V_Q9=4goJlw+>5{^C`#6oNv89Ky9e8hh-q)5EPOl%YYK$lG% z#Znv(N<76@e8uvhAQX(nT-?Qupu%1p#$sFxH9W>8?tx|+2Dabu=^~hrA>H{+JOY(uCR+O3SnOBlp1hfpQ+X!kK zZWU0oZ1h1MM+yi?D1$|T!EDE6qYsZVJ1(Xh4Zsw_-011z7-AvV8sh0M;wVuYdUx}J z1cpWp7M4YUO(%rH7ZrN;%*yCdi`!FW{PES%iti5&N`K6IbWnc5p@T;UtATd?{<;#^z?|eXrKmisJQ0S1Cq|Jb23ZQt(DW&taA+UQVL2 z&1cUzO*cEN=XWq(l$-5BHbZve^4klz{GE?VEi|9s^yx9ns>YczO?sW9Hetp^xrDSSMjpzSA91hLbKKbaO83R2RUKUc2`sk! zI}17g%`47ZW1iP4`F{D_wuF8AT2HNSS(nu}N%(a8v{!034*ort{!aUP?)jUC`L-Rl zFFNV#eN}5a!?yC41lbR-{H*ocnAay9zaO}Xk#R{`@tSoDS$8EQe_8nHj`f?o=5@A? zoJs4Novi=dIHzpJRsXz_CHA}EzI`5Ucl2IOn8JT}+vomucVrt=w;CtUzvH%k%`s1f zglE>7-;68odZx_Va-byc`O^)$+2$WrG;e=lD79eTx=)E|$~(g|3G5}V!6wuHmWt+E zwk-G@$Fl7q!(R)JIX|b?N*xq$GQAN0^Wu^@-m|Xh`8bK^cRibNr02^e*6h1cH|jW7 zRk*XT-Ev*G?akt+WkIh`WpbOvyfAw6h2`jjOpj+p2Y7jme>Ix>T58&tCNC=PEz*s) zob|M$Y1N+~bCCdX8&0kDE$UroCnl>-eCVoR)n}8W)OS|r`jIb}CdNNh4m|VbF3058 zGP0uQ7phuS?s+D7tdi3p=M}r^3?Vs{M^D2d<9gd Ndb;|#taD0e0sy+gVa)&l literal 0 HcmV?d00001 diff --git a/assets/orangebooks-logo.png b/assets/orangebooks-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..c2fbbabe1a9be80777f1a83e6e10ac9e8126bae0 GIT binary patch literal 10263 zcmV+yDCpOTP)su`XYX%+-#+_1()#>7U%72Z zdy*R<(ut_z@So{Hhyg7$&uc9yyzzM1;|+7XVU9P<@rF6xFvlC__)lk!6r^IrIF4d0 z1)Ulpi3CUcDr#{P1r9N(=RloE!L)DM4X8%C=@acI#uF2UCLjV|`Gf?1GEVA4DPJ++ zcO)k8yIi;b(#C?@n~UEaOdQNX>vh1miVlF*N+e1G149U5gi@_j-2sW>sq`WZ3JTOW z##c$MP#li*R1E^*Rokx5E7j0l=!gF4qdPmjURvocUj2u~?g2=0;L6wpozb%}b_9+- zkTQ53m?LWBC>6{}TwepFpj?r#q|Iq4e3s^z0D%r9l^aMmpOAcRnt0{@aPe)h;3g<> zDCYn=fIHoPwcc-y*4rvb1wa|Ow;@~*fZXy%Q-RJ&^?L9QQ}U$bsiMaI)eZs1h#P7R zZ7JhvQ$oH&)kyynJ=r+D$Xp`yj z(3BHaP7U`d^^ATUm?P1LIL$MtQqloR5@>3eBCJK)-Q2Hui#PLS?}ExwD4?$sXi@d_ z=MpuPMtci9Y9d}hjRB+J)N@sYAZ`6^Tw1M?=Y%=R!=H_bd=u%{g?E%x(cof|NyX}t zdjj&nmJ;xD{2=3om3m-sQ-cQ#5LvV;P!nF1Hnw@=W1x(~z%?Zzbm#yJmqGen5DARD z;4RQHL|n#z0T4|RHy~*PNnZz>3vd42{5$?Kb##_T0dw4pC&{If#xjD_Cv*L`y++Q_FOMS4w2vgtQGTp^iW>VgYgUfKRXV zdVo@W41$H_uFY)lEwJQDD9uY4Zv#|1*G(UKe)`Diu9XvB8NpHkp>}=et@B?9(#exh z9~I$h$}uS;=1}M(# z3D7c-5@T&xc)fNgcRS(0_s}oUhf;S$xIl8(x?WC`usAgp1NB>Zycyvo=vXdod8FZ^ zInTd29LGu1v=JqVB9aA);S4?I<$R}29aob+A$uE=H9{ZO-O_X0*Mq^gq@p3jv^o5w z?{yJ~APkrRuXky#IM9|VI<>=Q4`XQwMe5-AkYDPsRys9=*a6hI;E{}SBJ>(#q?C3U z=nM^|KVep?!7#^lyc~4`157C+QkBL@Fs+H2npCGJc(=FaBZV9PfM52}xNB3e=G_qH zshkqq?#DDBbv>pzf*GQ7G3~!x6Pk=ZW2T0HNJY>XE^x+suj0@YqkE90v<-3!k6=&` z@QRC!QE3t}(FBVt{~wTap@<<2Z%ZXH#I3-{0w;k{4k_zo(#eZOn{EiN{j^&A?xr*7 z;|CU8?hjn;OQnamk&}-q;tQK5A*Bv1y21Al4W`MF$G~RMhRqZPoCViOB53QltQ}UA zf%I55SFgI#pd(2^qnb5`maj0$Pa?ArhVDv4>_Qr5yjTh&6+a2(G+gA%a_i(NRX^24 zjPnC%L&9=w#qET5fvF`szTlD^Mc1Gj*Bj0_WKm<3)F2OR3-V3GM;gTpm z`k0IJmK0GzgLX@6Y-JSLerrlu3n7a!KVfrFbY==JBMxW6uql~q7FWQM&HZR$Xatd*0Qu_fphcBN%cbQIppy7y%*;wDMIZsiZG7hQ=hQbq=+OxaqqBkVV9`gR@0=a2d~Oq@pJr! z-+{tnckS)r%8P1K2lciGVR*zd5b*{#JPC7p!!5p7aiT`D>u&+3!MLdGB=WA+w^xm; zRVw#<3#W_2GXe?1#Qdvc8U&}eLaZ}+VlfNC`PLlh9nC5HEY6;opEP#JG~rg0a_#u} z`71jwxrwiO=a|17Ng-6$@kQ^2&Lyqp3o!aP-m8HMCIJj_y5uU#3XGL%yYB_kpaz3p z3YZqXn;aDWel9umXbNbe7~=x7H*{YRf# z@O$9UoepWpnVjkbL!H-F$haAM$W=~49AeS*yqLqZ88{q@D)wVcsDbXp9H`W()7~=E zbqQSYDedGYS_U5f3Pe#IFabD1JCHQI%RcVr=f(9C=9$058%!Vvtl=0CzF%DVUa6<_ z@ttn#C?{3?g!P|=tJ=X51djChgZMUZavrtmCEq=0;y9f+g+4%LAQso7(4mo=%OuOQ z^!#O(z__t$)=eac$MKHTlrWI`OA4#s8(jMvmrs3FA@WOpIterb^a zE)!%Zbv6Si=L<>`-zn#*PxS;QIo@@(zxEEe{CyBE;^I`zx0AcO-_vrt!=n$ufo}qMZICW1EJr!Z-`^^Bz=@}#BM&(EZd<{itZyypT25L) z>)1bXOsU3Jxu7s0N(+3|hT+|a-I#Gu_R`!!Nj#O}c%w*F^?5T#EF>|-RLLor)V&*u z>p$zQcz=y1ndg{1)EVlqz6)W~&#@qYz2AgG-vDTpbp!LTE7ExFC)oUJpgksbe$90Qfoq>gTRkpXH5 z`@t=MN}xTQ<=*pU4$ShgY6-94EV?PU@T0Kw3dJ&Hw7g(mAe+bzKsMC6UuN^Kt&3V? z&)>rMvq+7s!IBmYdh%=E9~T$E=+g1$sOjzcQPY!ga$rv zrRgY#lcO-S6=gHCw)28To4_Ol-w71cw!jz)di=1&jM9yh);HMs;vFs~6E>CW9jI*l zZMOL4rYjOPiK$UJ2yoEm^NQ)wI4Cf$-(UY>*-=g=c8Z;4q_E<$k3 zYZ9`ebOE6s6;)s};-GKSMg(0eo7enxr#zbM{}Swdz*l}9g%+O%$8$ITe%&4DigvcQ z|Ibj3E6+N&4%iX53JPBRr z`TJbf_D~_kQ@r!KmQxJsM}Q8;=UmaYX{6j-qH}SSdP;;0DKTxBr0a?g!HPFSz8^0s zOcp&a2C&Hx-DWP%DgUf#j1~xY!mt-NInS)|BWuPg^I1y$) zG>Jj;c`WGAoBnH9m@mhkge`wyB_84`v-uG=e1HVKYI0a?{cFu=sG7*QRKAaHe6OQ| z_Q8LE=_7#(QY7fzOL+fPU|QzT!>DHP8Upg};MK;6_Q|K^%%+m-Lv@j>VpSgnmS^YO z93E7Km||Yd$=kr5R*~*ooxkW;Aio?jAmcvx9s$%kLKdrvT3R3p(I}{-)L`4M0A;sSM)vMD4U9j&sf>oUc+hbrCoe z8LoQ?EVw4W>JFHHJrV-60A_?T%^KadV4xWtp3foAG@lV=1VTm|HN73G1$zpPox|Xg!`YU6AC)lBssj{K|%p z^F?oO6RS8~B7B|Ve=MwNI+Nxd8gSwojd!L^#)Hrr*&W5s!P!zHQ1jMhDe=7_0dc&rU`ikr%-J_t*%wfsV61P51< zAT$KXh%^gOTvMCe>|HVHScBN*()eDyDSaRA$WX(^rYEQAX4G!-|l=h)+s>|K3MNl77wPCyd2Hc)CDH8m#eehy_Bl%A2k?4Kq zH3tjTS|*M$j%FAE8kw}?H8H!XXuQdqD{g~pKLJ5VAmWt7%$8=YYNWP2WuG>;;tTnf zD@9g>qBIcT#dZUgqHOaJ5|q~#kRsk({FzCAk!n8TJo6btQ`h8kiKCt3*bV>PmAU+* zf8k8-OdP003JHe1{4?G}RXq1u&=WX5O6PR>uN8wa`RLzkzuIptJ+=+ zv~j@6tb>HWY)MmTCedif)Kg@Pwla=Xj%Y6gclfEPzxnlw^@Y)YRc*{}XLMHXpiu>*CT1lN_bf$80dT5OiN! z&`mbBOQnh%?e`B9sT)lkv&4}|Uq`-f1*ZKf%#rQfVCY$oXMp@r1Q>p4GG6J+t$q(I zcq`Z%+$2&45Do)9t4%EflOQ5YG!FWO-D(jS)6~X`V$a{zdf;P~q6?FgJ@Ik-_NPB* z>c=qT6>_bxN><(;1p{L5*VNFD(41To2p8z}x9W7NdHB0%RYX(p3wqH_B02;|A93s@ zqhP@eflm0T$6fB{a3<9O8hiEnVLL0KNu01GqDk{rm;-|?Nd$GRkiw{8$h6VG2@6)= z21~9&BeZQH7CpqX-Q3WgbyLy?u(Tn9UILCZ++hI*GFLUe|4ZYKf8I%tBty@eB5M4f zsRx72iT@4w$i+oMBwU#LvjIP{cNv5HFC0niIjx8`7h;zbsgQOmrlV4=vExZ`Rg@ri7* zCUf5Wja|z=1Yr>uD#e9Dl3}yChS|LXZ&zR}Iowd*K;*#5;CqorfA@C}fA4Ot8m_8E zZijp0@4?&tB;53oG~WeJeF2VtUxR0<4557WJ49(Q-TzH?Y#c0 ziy@9r1ns^g=-SjC*#cwRImxBeVJokr=sK-Y^zI};^1hO)mcn&2P*bc;f`sK>9d>Ze zkwGDiVw~56GaW4QF1{UdgTe~PnolX4+4iBRQTXiMVbvxEU1A_`!ScfTf!fn|o4x-K zq*D%fvg`8lhwn-jT&mn^wEhlf&Bt0JJLNNUYkHV_uYq=#8?vxuBMQSPJ9p3m<_$4EXEEqGZiu&U@{~! zmgZHM!(`%E$<%di?|}3=N-N(CORk@B9GnI}Q)$C#;thnelX~I+lS-iIw1M3?V0M04 z?D{{`-09=v(_|^!`gQ2Kgqucd=KUK!O4nW+pL~*R{Tc|21G{zRum7N0a&vlm8$A2v zltq~)EVGRtP|S<>eho8+o!H`Q+}v_=?E5Gb@!r(@O}k$K zuh2di(t7BsSLYo{p=)UFyFzNG)LmYGWt(*2{JC*C_}M;a1S|#OsySRtqepO`ISeCC zOx1ROd*b=~J7G^Q9d4B`?Z5Gp-0z!$ymE_0Gf@z6tI!i%@{5Qu@%Ar5^O#YbX)e7g zyXe0JUOKhy?u2HM*#`IG_lo@d@dMw4X#{Bjm4z(7!jFzc<3}x>s}AZIlA#|!Yix9Q zmsTx3a(MWW`-boPv!N~b!`LZDN9;2<7Pw)88RR-vdoxL9aT(16S5FcQUg_4;-s(I4 zq|tS8UM84+$x9`ATzG;%k}On`I8Bff5=t#W&GyVPMcgWkzKIT1O7 ztEE?vF~p zcGUKAKr$56$<(V!v>05(bCEA&zFU1A|4OF87nZw+&qICMa}RQV(`zm<>#_Q5YGSk5iWDg!=8sKN_=RalXz4IJ10Te&f$Tz86$e@>iA&Ue?Ef zL8>j^GU#zU<191Sp~3B=<2%HYchm7B#ZsQRtDVii3iB_pw$LbNW|~t_+F>%b?KvLW zr>8G>#i!DL`jueoNtnMG=G}lf7kbuKFS~u>;a?B8-ktlYO-PZ0unGeoiH|*|cKuO0 z)i$RN=Nbo!<*JDAPak(x;>HP0koRMbcfE>VsG8zODEA!~<$m}o%%Pdj)0z${?y|Q) z-+IsvrU*Kh(zOg->X;Eus5_H$&>~@b9DINJo1g3$JIX^8?Q!m>{!}bnkFam7wu4OH z>TH*Y+717#Lek{SU-}NV^6&K4hvb&OD%`vQ+-}6$+}d}sy$>fR9|&qU=R2=wdJmMP zcW`2Q-$4?m)lfqPRpxXb_ziyCFD>Dfi%HLAu;5ZC4}zC7groLg2sNw$(t8!=P}FB? zG%jBfuDb<{<(4T4E%}-yiOleH)NX{>Jwha+3`r=&_>0r`eXzGal8{&s7F_e+Y6F-1 znWbpf=VxWE!cl7d}#c@J^sjaJ{~UTxnOKs7iwOQQ|z19cR{%J7g$gQTC!r&wE-y%rldAfG8@iN zVdhmeVJn$TzS0$huJD78pQrH>kz4}uS>h&%+_4&M4RdH?Q;8_BI=8azV5k- zuKVSt=T@4NLEch(L?8hj%@(GEQpz z*+SPl>6O0)C0dp+nX6XKVNmlZT7Y|Ki_GOv9<~F|1L~!*fS^FgbT0K!u!rkDVD{bH z+IDx^xoCXLP68pzvn zaFTTdSl~ZVR#3-+%mzpd($r#@p}<(76`TfiMI+yuf0Y|BG~< z$yR}9TR)Mkgn=W~M8rB$7fN_Rr!e)l2fYWcR^Pp=FIKK25@8vU+3+s^lFt;o25o5u z1SWPC^Wy0CnTr&&#t!2Ulaxcy?WmI1+CPgqq9h4Wd{~)=BT|nO z#|zsj?^tvJI3?6SHV;}JuCxE~*~xe$Y1tAf5;jdWL*UJMJC;21Is zHbQ2~VrNfN=!lTk!0kXAmC~m1@%X>#h@Fye6Hg0W zH?HlY&)*9NzDZ;xgNl<*v%WX!t3Jhg*MLacP9 z|60sJcqTrmU4PFJ=p*ff1@qQG&uY}~2zD7LW=A0At}P05G7Jn_Fge)w*Jk*!uCOmA zL2={l(6vsHJetFEG7q7Q&0alg0DG~v##@bJdi$Sy&wP!Npp{=1pfFkews^xQc*kPf z6{1^VW1o8BPvemTdh7*X)dM#GFT}Kx;2{4V$$mC+E#lZP=20cMhm4Xool5tbw;Cua zd*d@U(AoHO?y>VATFisd(RAP4)Obvc!;%|=rB^DFYs;4;u!R;jM}9}=REc3;8nx-x zJDq+1*HFb(o50F+?aln+JHRbiTZmeEi??RQQDV4{Oy}TB4*d1~dW|}&V7bjCHD3up& zgxpfDvd+}Zagj@tY_o`|S*Kw;VUa+L9c&!EzrYjJOU|kfLEq&>X)9ndDlV0F@Z;Hr z%j_pOgd33hw!iRq-H95g#pk)@6?)}$%{A}KH;(&m2}Gm))E~B9xYuWH#c?AcwDDAm zE159Tpy+uB`G&x23l@>umcK+Q&*7=m4leNEs+drpO1rVBk#%D-ux??LJ)8!!peZxl zAdWqIs4>33kW->LR=Vi7ngsF&xrns8*P0-IH(5{49AT&2kZrw7@B9))B8h8citg%n zcpGk*!e12@uzf#LkN#1+w#~^qTyfcScsSy)0{#wVt1v<{nQ)PE5lZ(p1xvET^G~

vK|?PHgg?x zaR6y2+Qf^fJ+w&DRSh3!+Iks{G- zEy$TY*mGzON+N;ACa&FN?dX8&o2)x)I=k?n*@aP*oY@1leNH&L!&~^oj*{mKB&V0$ zg6zIcTt4SU4$&^+NE=}g--$f?*dx=s{}D)7n3kgd&Ee(0N%GYg#?$7J_7lGoPCpl7 z=C)qzrJ85zT-%n8bL<`VdRxm~D*Addf52jdBB`_P*~pxK^q~ z2CjgfOCU}o@Q9uR$HdHB6m5p!L*saT_t*KFt_aWUSUwXO2(YoH3gfgeQ5*`HPgm`GQ56V zpL&Myf@(GSML*kwa-Jt{)(=eBY(ZzWJfRwiBoPVh`{(%B4?7FxX`Om2Z&QQon9?R` z#7BPUZ+}R$a;&2?>zL4(^%?a(6Yd?{;k1PkL_n7?;<A4s5;)xy;r zLs$&6eKqt9B&Lnp@x7Etn{fM=rMx%CFl$n1j7l8-V-XkLQuwP3~Z3^%cjrgI2a8&cFWFZ_eA zr@^nHQijz(jecsq?&wx^awp|6akxC|X3!6`pa=0HM)-odMj>|umVeH%#Tz(CuwW)G)|)PBJMPfzZq49Td@m3w5k zAe(+h71a!3oZZRs*GHaA;Dmufnu_A`cL8-c6)<%m9eW0dpK>SGsBi;EPf$b8pE+#3 zBv;jv>KDs}M$gegm(_(#qOuQ*sS^~bm=Xoa_G!ERH%i5NIogZ^>6D24^`>z2m3lK3XX}nOec(x&2 zQ();t)5+%{f@qu#9mvHd^3usAKVM3wa=5(O$<7{y$VkQPjWA58`qa~vX@^Cl?%>9> z&`)V<(DkZ>t2O5iJLi^2+=X+S_ECmer#a~gAiHU!sv}RimX0l}+MTc!C zCVm*^&y=rcY7jF!*mEd~#tD;5CRW%ow`aNSw?`(TrnQuMrf^1Sb|7|!8J#chK-_cO zqE%Yi;oLL4V{TzYoimlE#z?jm56Uh2xw18f-J+ct0Y|efFS8CZ{ya-ZWA1C>ID?nc ztXW%~+?S@Q!VLVPIn0ZX^JNZ3+@NgbWmdz~XUsw8g21dyXutZcHe`+ zseu<;dCctQP-DAnN4XQkJP*C=5u5GVjbl4z8q!`d{~0$qALdY){exE~V^ATtk= zJ)OlQ(bwkbjA9HmS( zKBNuvbfQZG_FWFfX_GdOy&^__Ooo_U!dZr1%HZ>*!+Z+n3{^FA@(*X5Od^!x9Xt7M z8$haBa&HSj=7L%S{Tnu+g{hwGuxu^n33Zzpa09z^t?f-{pzSJNZ`!B_}Uyx zGIR2ACjY4ROi)m_V91%_tqOhV6wU18I0lK1hx`R~Ft`0U0Iew_TQA3pm%>E$SSPqj(F9iH3uWp(t?my=NOuWJY| zN5mPVjGa~ype;+vxVFeb&?HD-NOoQeUf~q>OIps4IjUj=>Qkc5g+}&7xZQ`<22qur zT~jMBZ3%}c&I!_wIM9kJ2bG0vZJFc3FCpR#BG^)^7e>v<4fA2OYK<%<>NJDTA5UDI d_3{4%7ytr;(6Wg65>NmD002ovPDHLkV1kU797g~E literal 0 HcmV?d00001 diff --git a/cmds/csv2mdtable/csv2mdtable.go b/cmds/csv2mdtable/csv2mdtable.go new file mode 100644 index 0000000..eda38e4 --- /dev/null +++ b/cmds/csv2mdtable/csv2mdtable.go @@ -0,0 +1,140 @@ +// +// csv2mdtable - is a command line that takes CSV input from stdin and +// writes out a Github Flavored Markdown table. +// +// @author R. S. Doiel, +// +// Copyright (c) 2017, Caltech +// All rights not granted herein are expressly reserved by Caltech. +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +package main + +import ( + "encoding/csv" + "flag" + "fmt" + "io" + "os" + "path" + "strings" + + // My packages + "github.com/caltechlibrary/cli" + "github.com/caltechlibrary/datatools" +) + +var ( + usage = `USAGE: %s [OPTIONS]` + + description = ` +SYNOPSIS + +%s reads CSV from stdin and writes a Github Flavored Markdown +table to stdout. +` + + examples = ` +EXAMPLES + +Convert data1.csv to data1.md using Unix pipes. + + cat data1.csv | %s > data1.md + +Convert data1.csv to data1.md using options. + + %s -i data1.csv -o data1.md +` + + // Standard Options + showHelp bool + showLicense bool + showVersion bool + inputFName string + outputFName string +) + +func init() { + // Standard Options + flag.BoolVar(&showHelp, "h", false, "display help") + flag.BoolVar(&showHelp, "help", false, "display help") + flag.BoolVar(&showLicense, "l", false, "display license") + flag.BoolVar(&showLicense, "license", false, "display license") + flag.BoolVar(&showVersion, "v", false, "display version") + flag.BoolVar(&showVersion, "version", false, "display version") + flag.StringVar(&inputFName, "i", "", "input filename") + flag.StringVar(&inputFName, "input", "", "input filename") + flag.StringVar(&outputFName, "o", "", "output filename") + flag.StringVar(&outputFName, "output", "", "output filename") +} + +func main() { + appName := path.Base(os.Args[0]) + flag.Parse() + + // Configuration and command line interation + cfg := cli.New(appName, appName, fmt.Sprintf(datatools.LicenseText, appName, datatools.Version), datatools.Version) + cfg.UsageText = fmt.Sprintf(usage, appName) + cfg.DescriptionText = fmt.Sprintf(description, appName) + cfg.ExampleText = fmt.Sprintf(examples, appName, appName) + + if showHelp == true { + fmt.Println(cfg.Usage()) + os.Exit(0) + } + + if showLicense == true { + fmt.Println(cfg.License()) + os.Exit(0) + } + + if showVersion == true { + fmt.Println(cfg.Version()) + os.Exit(0) + } + + in, err := cli.Open(inputFName, os.Stdin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + defer cli.CloseFile(inputFName, in) + + out, err := cli.Create(outputFName, os.Stdout) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + defer cli.CloseFile(outputFName, out) + + r := csv.NewReader(in) + writeHeader := true + for { + record, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + fmt.Fprintf(out, "| %s |\n", strings.Join(record, " | ")) + if writeHeader == true { + headerRow := []string{} + for _, rec := range record { + headerRow = append(headerRow, strings.Repeat("-", len(rec))) + } + fmt.Fprintf(out, "| %s |\n", strings.Join(headerRow, " | ")) + writeHeader = false + } + } +} diff --git a/cmds/csv2xlsx/csv2xlsx.go b/cmds/csv2xlsx/csv2xlsx.go new file mode 100644 index 0000000..b081f77 --- /dev/null +++ b/cmds/csv2xlsx/csv2xlsx.go @@ -0,0 +1,184 @@ +// +// csv2xlsx is a command line utility that will convert a CSV file and insert it +// into a named sheet in an Excel Workbook. +// +// @Author R. S. Doiel, +// +// Copyright (c) 2017, Caltech +// All rights not granted herein are expressly reserved by Caltech. +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +package main + +import ( + "encoding/csv" + "flag" + "fmt" + "io" + "os" + "path" + + // CaltechLibrary packages + "github.com/caltechlibrary/cli" + "github.com/caltechlibrary/datatools" + + // 3rd Party packages + "github.com/tealeg/xlsx" +) + +var ( + usage = `USAGE: %s [OPTIONS] WORKBOOK_NAME SHEET_NAME` + + description = ` +SYNOPSIS + +%s will take CSV input and create a new sheet in an Excel Workbook. +If the Workbook does not exist then it is created. +` + + examples = ` +EXAMPLE + + %s -i data.csv MyWorkbook.xlsx 'My worksheet' + +This creates a new 'My worksheet' in the Excel Workbook +called 'MyWorkbook.xlsx' with the contents of data.csv. + + cat data.csv | %s MyWorkbook.xlsx 'My worksheet' + +This does the same but the contents of data.csv are piped into +the workbook's sheet. +` + + // Standard Options + showHelp bool + showLicense bool + showVersion bool + inputFName string + + // App Specific Options + workbookName string + sheetName string +) + +func csv2XLSXSheet(in *os.File, workbookName string, sheetName string) error { + var workbook *xlsx.File + + // Open the workbook + if _, err := os.Stat(workbookName); os.IsNotExist(err) == true { + workbook = xlsx.NewFile() + } else { + workbook, err = xlsx.OpenFile(workbookName) + if err != nil { + return err + } + } + // Create our worksheet + sheet, err := workbook.AddSheet(sheetName) + if err != nil { + return err + } + + r := csv.NewReader(in) + for { + record, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + } + row := sheet.AddRow() + for _, val := range record { + cell := row.AddCell() + cell.Value = val + } + } + // Now write out the changes + return workbook.Save(workbookName) +} + +func init() { + // Standard Options + flag.BoolVar(&showHelp, "h", false, "display help") + flag.BoolVar(&showHelp, "help", false, "display help") + flag.BoolVar(&showLicense, "l", false, "display license") + flag.BoolVar(&showLicense, "license", false, "display license") + flag.BoolVar(&showVersion, "v", false, "display version") + flag.BoolVar(&showVersion, "version", false, "display version") + flag.StringVar(&inputFName, "i", "", "input filename (CSV content)") + flag.StringVar(&inputFName, "input", "", "input filename (CSV content)") + + // App Specific Options + flag.StringVar(&workbookName, "workbook", "", "Workbook name") + flag.StringVar(&sheetName, "sheet", "", "Sheet name to create/replace") +} + +func main() { + appName := path.Base(os.Args[0]) + flag.Parse() + args := flag.Args() + + // Configuration and command line interation + cfg := cli.New(appName, appName, fmt.Sprintf(datatools.LicenseText, appName, datatools.Version), datatools.Version) + cfg.UsageText = fmt.Sprintf(usage, appName) + cfg.DescriptionText = fmt.Sprintf(description, appName) + cfg.ExampleText = fmt.Sprintf(examples, appName, appName) + + if showHelp == true { + fmt.Println(cfg.Usage()) + os.Exit(0) + } + + if showLicense == true { + fmt.Println(cfg.License()) + os.Exit(0) + } + + if showVersion == true { + fmt.Println(cfg.Version()) + os.Exit(0) + } + + in, err := cli.Open(inputFName, os.Stdin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + defer cli.CloseFile(inputFName, in) + + if len(workbookName) == 0 && len(args) > 0 { + workbookName = args[0] + if len(args) > 1 { + args = args[1:] + } else { + args = []string{} + } + } + if len(sheetName) == 0 && len(args) > 0 { + sheetName = args[0] + } + + if len(workbookName) == 0 { + fmt.Fprintln(os.Stderr, "Missing workbook name") + os.Exit(1) + } + if len(sheetName) == 0 { + fmt.Fprintln(os.Stderr, "Missing sheet name") + os.Exit(1) + } + err = csv2XLSXSheet(in, workbookName, sheetName) + if err != nil { + fmt.Fprintf(os.Stderr, "%s", err) + os.Exit(1) + } +} diff --git a/cmds/csvcols/csvcols.go b/cmds/csvcols/csvcols.go new file mode 100644 index 0000000..2901721 --- /dev/null +++ b/cmds/csvcols/csvcols.go @@ -0,0 +1,221 @@ +// +// csvcols - is a command line that takes each argument in order and outputs a line in CSV format. +// It can also take a delimiter and line of text splitting it into a CSV formatted set of columns. +// +// @author R. S. Doiel, +// +// Copyright (c) 2017, Caltech +// All rights not granted herein are expressly reserved by Caltech. +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +package main + +import ( + "encoding/csv" + "flag" + "fmt" + "io" + "log" + "os" + "path" + "strconv" + "strings" + + // My packages + "github.com/caltechlibrary/cli" + "github.com/caltechlibrary/datatools" +) + +var ( + usage = `USAGE: %s [OPTIONS] ARGS_AS_COLS` + + description = ` +SYNOPSIS + +%s converts a set of command line args into columns output in CSV format. +It can also be used to filter input CSV and rendering only the column numbers +listed on the commandline. +` + + examples = ` +EXAMPLES + +Simple usage of building a CSV file one row at a time. + + %s one two three > 3col.csv + %s 1 2 3 >> 3col.csv + cat 3col.csv + +Example parsing a pipe delimited string into a CSV line + + %s -d "|" "one|two|three" > 3col.csv + %s -delimiter "|" "1|2|3" >> 3col.csv + cat 3col.csv + +Filter a 10 column CSV file for columns 0,3,5 (left most column is number zero) + + cat 10col.csv | csvcols -f 0 3 5 > 3col.csv + +Filter a 10 columns CSV file for columns 0,3,5 from input file + + %s -i 10col.csv -f 0 3 5 > 3col.csv +` + + // Standard Options + showHelp bool + showLicense bool + showVersion bool + inputFName string + outputFName string + + // App Options + delimiter string + filterColumns bool +) + +func selectedColumns(record []string, columnNos []int) []string { + result := []string{} + l := len(record) + for _, col := range columnNos { + if col >= 0 && col < l { + result = append(result, record[col]) + } else { + // If we don't find the column, story an empty string + result = append(result, "") + } + } + return result +} + +func CSVColumns(in *os.File, out *os.File, columnNos []int) { + r := csv.NewReader(in) + w := csv.NewWriter(out) + for { + rec, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + fmt.Fprintf(os.Stderr, "%s, %s\n", inputFName, err) + fmt.Fprintf(os.Stderr, "%T %+v\n", rec, rec) + //os.Exit(1) + } + row := selectedColumns(rec, columnNos) + if err := w.Write(row); err != nil { + fmt.Fprintf(os.Stderr, "Error writing record to csv: %s\n", err) + fmt.Fprintf(os.Stderr, "Row %T %+v\n", row, row) + os.Exit(1) + } + } + w.Flush() + if err := w.Error(); err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } +} + +func init() { + // Basic Options + flag.BoolVar(&showHelp, "h", false, "display help") + flag.BoolVar(&showHelp, "help", false, "display help") + flag.BoolVar(&showLicense, "l", false, "display license") + flag.BoolVar(&showLicense, "license", false, "display license") + flag.BoolVar(&showVersion, "v", false, "display version") + flag.BoolVar(&showVersion, "version", false, "display version") + flag.StringVar(&inputFName, "i", "", "input filename") + flag.StringVar(&inputFName, "input", "", "input filename") + flag.StringVar(&outputFName, "o", "", "output filename") + flag.StringVar(&outputFName, "output", "", "output filename") + + // App Options + flag.StringVar(&delimiter, "d", "", "set delimiter for conversion") + flag.StringVar(&delimiter, "delimiter", "", "set delimiter for conversion") + flag.BoolVar(&filterColumns, "f", false, "filter CSV input for columns requested") + flag.BoolVar(&filterColumns, "filter-columns", false, "filter CSV input for columns requested") +} + +func main() { + appName := path.Base(os.Args[0]) + flag.Parse() + args := flag.Args() + + // Configuration and command line interation + cfg := cli.New(appName, appName, fmt.Sprintf(datatools.LicenseText, appName, datatools.Version), datatools.Version) + cfg.UsageText = fmt.Sprintf(usage, appName) + cfg.DescriptionText = fmt.Sprintf(description, appName) + cfg.ExampleText = fmt.Sprintf(examples, appName, appName, appName, appName, appName) + + if showHelp == true { + fmt.Println(cfg.Usage()) + os.Exit(0) + } + + if showLicense == true { + fmt.Println(cfg.License()) + os.Exit(0) + } + + if showVersion == true { + fmt.Println(cfg.Version()) + os.Exit(0) + } + + in, err := cli.Open(inputFName, os.Stdin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + defer cli.CloseFile(inputFName, in) + + out, err := cli.Create(outputFName, os.Stdout) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + defer cli.CloseFile(outputFName, out) + + if filterColumns == true { + columnNos := []int{} + for _, arg := range args { + i, err := strconv.Atoi(arg) + if err != nil { + fmt.Fprintf(os.Stderr, "Expected a column number (0 - n), %q, %s\n", arg, err) + os.Exit(1) + } + columnNos = append(columnNos, i) + } + CSVColumns(in, out, columnNos) + os.Exit(0) + } + + if len(delimiter) > 0 && len(args) == 1 { + args = strings.Split(args[0], delimiter) + } + + // Clean up fields removing outer quotes if necessary + fields := []string{} + for _, val := range args { + if strings.HasPrefix(val, "\"") && strings.HasSuffix(val, "\"") { + val = strings.TrimPrefix(strings.TrimSuffix(val, "\""), "\"") + } + fields = append(fields, strings.TrimSpace(val)) + } + + w := csv.NewWriter(out) + if err := w.Write(fields); err != nil { + log.Fatalf("error wrint args as csv, %s", err) + } + w.Flush() + if err := w.Error(); err != nil { + log.Fatal(err) + } +} diff --git a/cmds/csvjoin/csvjoin.go b/cmds/csvjoin/csvjoin.go new file mode 100644 index 0000000..6d2d01a --- /dev/null +++ b/cmds/csvjoin/csvjoin.go @@ -0,0 +1,211 @@ +// +// csvjoin - is a command line that takes two CSV files and joins them by match a designated column in each. +// +// @author R. S. Doiel, +// +// Copyright (c) 2017, Caltech +// All rights not granted herein are expressly reserved by Caltech. +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +package main + +import ( + "bytes" + "encoding/csv" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path" + + // My packages + "github.com/caltechlibrary/cli" + "github.com/caltechlibrary/datatools" +) + +var ( + usage = `USAGE: %s [OPTIONS] CSV1 CSV2 COL1 COL2` + + description = ` +SYNOPSIS + +%s outputs CSV content based on two CSV files with matching column values. +Each CSV input file has a designated column to match on. The values are +compared as strings. +` + + examples = ` +EXAMPLES + +Simple usage of building a merged CSV file from data1.csv +and data2.csv where column 1 in data1.csv matches the value in +column 3 of data2.csv with the results being written to +merged-data.csv.. + + %s -csv1=data1.csv -col1=1 \ + -csv2=data2.csv -col2=3 \ + -output=merged-data.csv +` + + // Standard Options + showHelp bool + showLicense bool + showVersion bool + outputFName string + + // App Options + csv1FName string + csv2FName string + col1 int + col2 int +) + +func scanTable(table [][]string, col2 int, val string) ([]string, bool) { + for _, row := range table { + if col2 < len(row) && row[col2] == val { + return row, true + } + } + return []string{}, false +} + +func init() { + // Basic Options + flag.BoolVar(&showHelp, "h", false, "display help") + flag.BoolVar(&showHelp, "help", false, "display help") + flag.BoolVar(&showLicense, "l", false, "display license") + flag.BoolVar(&showLicense, "license", false, "display license") + flag.BoolVar(&showVersion, "v", false, "display version") + flag.BoolVar(&showVersion, "version", false, "display version") + flag.StringVar(&outputFName, "o", "", "output filename") + flag.StringVar(&outputFName, "output", "", "output filename") + + // App Options + flag.StringVar(&csv1FName, "csv1", "", "first CSV filename") + flag.StringVar(&csv2FName, "csv2", "", "second CSV filename") + flag.IntVar(&col1, "col1", 0, "column to on join on in first CSV file") + flag.IntVar(&col2, "col2", 0, "column to on join on in second CSV file") +} + +func main() { + appName := path.Base(os.Args[0]) + flag.Parse() + + // Configuration and command line interation + cfg := cli.New(appName, appName, fmt.Sprintf(datatools.LicenseText, appName, datatools.Version), datatools.Version) + cfg.UsageText = fmt.Sprintf(usage, appName) + cfg.DescriptionText = fmt.Sprintf(description, appName) + cfg.ExampleText = fmt.Sprintf(examples, appName) + + if showHelp == true { + fmt.Println(cfg.Usage()) + os.Exit(0) + } + + if showLicense == true { + fmt.Println(cfg.License()) + os.Exit(0) + } + + if showVersion == true { + fmt.Println(cfg.Version()) + os.Exit(0) + } + + // NOTE: we don't setup inputFName as we need at least two inputs to process the join. + out, err := cli.Create(outputFName, os.Stdout) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + defer cli.CloseFile(outputFName, out) + + if len(csv1FName) == 0 { + fmt.Fprintln(os.Stderr, "Missing first CSV filename") + os.Exit(1) + } + + if len(csv2FName) == 0 { + fmt.Fprintln(os.Stderr, "Missing second CSV filename") + os.Exit(1) + } + + if col1 < 0 { + fmt.Fprintf(os.Stderr, "Cannot use a negative column index %d\n", col1) + os.Exit(1) + } + if col2 < 0 { + fmt.Fprintf(os.Stderr, "Cannot use a negative column index %d\n", col2) + os.Exit(1) + } + + // Read in CSV1 and CSV2 then iterate over CSV1 output rows that have + // matching column's value + src1, err := ioutil.ReadFile(csv1FName) + if err != nil { + fmt.Fprintf(os.Stderr, "Can't read %s, %s\n", csv1FName, err) + os.Exit(1) + } + src2, err := ioutil.ReadFile(csv2FName) + if err != nil { + fmt.Fprintf(os.Stderr, "Can't read %s, %s\n", csv2FName, err) + os.Exit(1) + } + csv1 := csv.NewReader(bytes.NewReader(src1)) + csv1Table := [][]string{} + for { + record, err := csv1.Read() + if err == io.EOF { + break + } + if err != nil { + fmt.Fprintf(os.Stderr, "%s, %s\n", csv1FName, err) + fmt.Fprintf(os.Stderr, "%T %+v\n", record, record) + } + csv1Table = append(csv1Table, record) + } + csv2 := csv.NewReader(bytes.NewReader(src2)) + csv2Table := [][]string{} + for { + record, err := csv2.Read() + if err == io.EOF { + break + } + if err != nil { + fmt.Fprintf(os.Stderr, "%s, %s\n", csv2FName, err) + fmt.Fprintf(os.Stderr, "%T %+v\n", record, record) + } + csv2Table = append(csv2Table, record) + } + + w := csv.NewWriter(out) + val := "" + for _, rowA := range csv1Table { + if col1 < len(rowA) { + val = rowA[col1] + // Name see if we find matching row in table 2 + if rowB, ok := scanTable(csv2Table, col2, val); ok == true { + // We have + combinedRows := append(rowA, rowB...) + if err := w.Write(combinedRows); err != nil { + log.Fatalf("error wrint args as csv, %s", err) + } + } + } + } + w.Flush() + if err := w.Error(); err != nil { + log.Fatal(err) + } +} diff --git a/cmds/jsoncols/jsoncols.go b/cmds/jsoncols/jsoncols.go new file mode 100644 index 0000000..c313ad3 --- /dev/null +++ b/cmds/jsoncols/jsoncols.go @@ -0,0 +1,199 @@ +// +// jsonpath is a command line tool for filter JSON data from standard in or specified files. +// It was inspired by [jq](https://github.com/stedolan/jq) and [jid](https://github.com/simeji/jid). +// +// @author R. S. Doiel, +// +package main + +import ( + "flag" + "fmt" + "os" + "path" + + // 3rd Party packages + // "github.com/simeji/jid" + "github.com/rsdoiel/jid" + + // My Packages + "github.com/caltechlibrary/cli" + "github.com/caltechlibrary/datatools" +) + +var ( + usage = `USAGE: %s [OPTIONS] [EXPRESSION] [INPUT_FILENAME] [OUTPUT_FILENAME]` + + description = ` +SYSNOPSIS + +%s provides for both interactive exploration of JSON structures like jid +and command line scripting flexibility for data extraction into delimited +columns. This is helpful in flattening content extracted from JSON blobs. +The default delimiter for each value extracted is a comma. This can be +overridden with an option. + ++ EXPRESSION can be an empty stirng or dot notation for an object's path ++ INPUT_FILENAME is the filename to read or a dash "-" if you want to + explicity read from stdin + + if not provided then %s reads from stdin ++ OUTPUT_FILENAME is the filename to write or a dash "-" if you want to + explicity write to stdout + + if not provided then %s write to stdout +` + + examples = ` +EXAMPLES + +If myblob.json contained + +{"name": "Doe, Jane", "email":"jane.doe@example.org", "age": 42} + +Getting just the name could be done with + + %s -i myblob.json .name + +This would yeild + + "Doe, Jane" + +Flipping .name and .age into pipe delimited columns is as +easy as listing each field in the expression inside a +space delimited string. + + %s -i myblob.json -d\| .name .age + +This would yeild + + "Doe, Jane"|42 + +You can also pipe JSON data in. + + cat myblob.json | %s .name .email .age + +Would yield + + "Doe, Jane",jane.doe@xample.org,42 +` + + // Basic Options + showHelp bool + showLicense bool + showVersion bool + inputFName string + outputFName string + + // Application Specific Options + monochrome bool + runInteractive bool + delimiter = "," + expressions []string +) + +func init() { + // Basic Options + flag.BoolVar(&showHelp, "h", false, "display help") + flag.BoolVar(&showLicense, "l", false, "display license") + flag.BoolVar(&showVersion, "v", false, "display version") + flag.StringVar(&inputFName, "i", "", "input filename") + flag.StringVar(&inputFName, "input", "", "input filename") + flag.StringVar(&outputFName, "o", "", "output filename") + flag.StringVar(&outputFName, "output", "", "output filename") + + // Application Specific Options + flag.BoolVar(&monochrome, "m", false, "display output in monochrome") + flag.BoolVar(&runInteractive, "r", false, "run interactively") + flag.BoolVar(&runInteractive, "repl", false, "run interactively") + flag.StringVar(&delimiter, "d", delimiter, "set the delimiter for multi-field output") +} + +func main() { + appName := path.Base(os.Args[0]) + flag.Parse() + args := flag.Args() + + // Configuration and command line interation + cfg := cli.New(appName, "DATATOOLS", fmt.Sprintf(datatools.LicenseText, appName, datatools.Version), datatools.Version) + cfg.UsageText = fmt.Sprintf(usage, appName) + cfg.DescriptionText = fmt.Sprintf(description, appName, appName, appName) + cfg.ExampleText = fmt.Sprintf(examples, appName, appName, appName) + + //NOTE: Need to handle JSONQUERY_MONOCHROME setting + monochrome = cfg.MergeEnvBool("monochrome", monochrome) + + if showHelp == true { + fmt.Println(cfg.Usage()) + os.Exit(0) + } + + if showLicense == true { + fmt.Println(cfg.License()) + os.Exit(0) + } + + if showVersion == true { + fmt.Println(cfg.Version()) + os.Exit(0) + } + + in, err := cli.Open(inputFName, os.Stdin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + defer cli.CloseFile(inputFName, in) + + out, err := cli.Create(outputFName, os.Stdout) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + defer cli.CloseFile(outputFName, out) + + // Handle ordered args to get expressions for each column output. + for _, arg := range args { + if len(arg) == 0 { + arg = "." + } + expressions = append(expressions, arg) + } + // Make sure we have a default expression to run. + if len(expressions) == 0 { + expressions = []string{"."} + } + + // Configure the jid engine + engineAttributes := &jid.EngineAttribute{ + DefaultQuery: ".", + Monochrome: monochrome, + } + + // Run the jid engine appropriately + engine, err := jid.NewEngine(in, engineAttributes) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + var result jid.EngineResultInterface + + if runInteractive == true { + result = engine.Run() + if err := result.GetError(); err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + fmt.Fprintf(out, "%s", result.GetContent()) + } else { + for i, qry := range expressions { + result = engine.EvalString(qry) + if err := result.GetError(); err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + if i > 0 { + fmt.Fprintf(out, "%s", delimiter) + } + fmt.Fprintf(out, "%s", result.GetContent()) + } + } +} diff --git a/cmds/jsonrange/jsonrange.go b/cmds/jsonrange/jsonrange.go new file mode 100644 index 0000000..2ca00db --- /dev/null +++ b/cmds/jsonrange/jsonrange.go @@ -0,0 +1,284 @@ +// +// jsonrange iterates over an array or map returning either a JSON expression +// or map keep to stdout +// +// @Author R. S. Doiel, +// +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "os" + "path" + "strings" + + // 3rd Party Captech + "github.com/rsdoiel/jid" // this is a fork of github.com/simeji/jid, adds Eval() and EvalString() + + // CaltechLibrary Packages + "github.com/caltechlibrary/cli" + "github.com/caltechlibrary/datatools" +) + +var ( + usage = `USAGE: %s [OPTIONS] JSON_EXPRESSION ` + + description = ` +SYSNOPSIS + +%s turns either the JSON expression that is a map or array into delimited +elements suitable for processing in a "for" style loop in Bash. If the +JSON expression is an array then the elements of the array are returned else +if the expression is a map/object then the keys or attribute names are turned. + ++ EXPRESSION can be an empty string contains a JSON array or map. +` + + examples = ` +EXAMPLES + +Working with a map + + %s '{"name": "Doe, Jane", "email":"jane.doe@example.org", "age": 42}' + +This would yield + + name + email + age + +Working with an array + + %s '["one", 2, {"label":"three","value":3}]' + +would yield + + one + 2 + {"label":"three","value":3} + +Checking the length of a map or array + + %s -length '["one","two","three"]' + +would yield + + 3 + +Limitting the number of items returned + + %s -limit 2 '[1,2,3,4,5]' + +would yield + + 1 + 2 + +Likewise you can have the JSON expression read from stnin + + echo '[1,2,3,4,5]' | %s -limit 2 + +would yield + + 1 + 2 +` + + // Basic Options + showHelp bool + showLicense bool + showVersion bool + inputFName string + outputFName string + + // Application Specific Options + showLength bool + delimiter = "\n" + limit int + dotPath string +) + +func srcKeys(inSrc string, limit int) ([]string, error) { + data := map[string]interface{}{} + if err := json.Unmarshal([]byte(inSrc), &data); err != nil { + return nil, err + } + result := []string{} + i := 0 + for keys := range data { + result = append(result, keys) + if limit > 0 && i == limit { + return result, nil + } + i++ + } + return result, nil +} + +func srcVals(inSrc string, limit int) ([]string, error) { + data := []interface{}{} + if err := json.Unmarshal([]byte(inSrc), &data); err != nil { + return nil, err + } + result := []string{} + for i, val := range data { + outSrc, err := json.Marshal(val) + if err != nil { + return nil, err + } + result = append(result, fmt.Sprintf("%s", outSrc)) + if limit != 0 && i == limit { + return result, nil + } + } + return result, nil +} + +func init() { + // Standard Options + flag.BoolVar(&showHelp, "h", false, "display help") + flag.BoolVar(&showLicense, "l", false, "display license") + flag.BoolVar(&showVersion, "v", false, "display version") + flag.StringVar(&inputFName, "i", "", "read JSON from file") + flag.StringVar(&inputFName, "input", "", "read JSON from file") + flag.StringVar(&outputFName, "o", "", "write to output file") + flag.StringVar(&outputFName, "output", "", "write to output file") + + // Application Options + flag.BoolVar(&showLength, "length", false, "return the number of keys or values") + flag.StringVar(&delimiter, "d", "\n", "set delimiter for range output") + flag.StringVar(&delimiter, "delimiter", "\n", "set delimiter for range output") + flag.IntVar(&limit, "limit", 0, "limit the number of items output") + flag.StringVar(&dotPath, "p", "", "range on given dot path") + flag.StringVar(&dotPath, "dotpath", "", "range on given dot path") +} + +func getLength(inSrc string) (int, error) { + if strings.HasPrefix(inSrc, "{") { + data := map[string]interface{}{} + if err := json.Unmarshal([]byte(inSrc), &data); err != nil { + return 0, err + } + return len(data), nil + } + data := []interface{}{} + if err := json.Unmarshal([]byte(inSrc), &data); err != nil { + return 0, err + } + return len(data), nil +} + +func main() { + appName := path.Base(os.Args[0]) + flag.Parse() + + args := flag.Args() + + // Configuration and command line interation + cfg := cli.New(appName, "DATATOOLS", fmt.Sprintf(datatools.LicenseText, appName, datatools.Version), datatools.Version) + cfg.UsageText = fmt.Sprintf(usage, appName) + cfg.DescriptionText = fmt.Sprintf(description, appName) + cfg.ExampleText = fmt.Sprintf(examples, appName, appName, appName, appName, appName) + + if showHelp == true { + fmt.Println(cfg.Usage()) + os.Exit(0) + } + + if showLicense == true { + fmt.Println(cfg.License()) + os.Exit(0) + } + + if showVersion == true { + fmt.Println(cfg.Version()) + } + + in, err := cli.Open(inputFName, os.Stdin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + defer cli.CloseFile(inputFName, in) + + out, err := cli.Create(outputFName, os.Stdout) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + defer cli.CloseFile(outputFName, out) + + // OK, let's see what keys/values we're going to output... + src := "" + if len(args) == 0 { + lines, err := cli.ReadLines(in) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + src = strings.Join(lines, "\n") + } else { + // If nothing is coming from stdin or a file then get the string from the command line + if strings.HasPrefix(args[0], "'") == true { + src = strings.Trim(args[0], "'") + } else if strings.HasPrefix(args[0], "\"") == true { + src = strings.Trim(args[0], "\"") + } else { + src = args[0] + } + } + + // If a dotPath is privided extract the desired field for range. + if len(dotPath) > 0 { + buf := bytes.NewBufferString(src) + ea := &jid.EngineAttribute{ + DefaultQuery: ".", + Monochrome: true, + } + e, err := jid.NewEngine(buf, ea) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + result := e.EvalString(dotPath) + + if err := result.GetError(); err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + src = result.GetContent() + } + if len(src) == 0 { + fmt.Println(cfg.Usage()) + os.Exit(1) + } + switch { + case showLength: + l, err := getLength(src) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + fmt.Fprintf(out, "%d", l) + case strings.HasPrefix(src, "{"): + elems, err := srcKeys(src, limit-1) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + fmt.Fprintln(out, strings.Join(elems, delimiter)) + case strings.HasPrefix(src, "["): + elems, err := srcVals(src, limit-1) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + fmt.Fprintln(out, strings.Join(elems, delimiter)) + default: + fmt.Fprintf(os.Stderr, "Cannot iterate over %q\n", src) + os.Exit(1) + } +} diff --git a/cmds/xlsx2csv/xlsx2csv.go b/cmds/xlsx2csv/xlsx2csv.go new file mode 100644 index 0000000..6568ee1 --- /dev/null +++ b/cmds/xlsx2csv/xlsx2csv.go @@ -0,0 +1,219 @@ +// +// xlsx2csv.go is a command line utility that converts individual +// Excel Workbook Sheets to CSV. +// +// @Author R. S. Doiel +// +// Copyright (c) 2017, Caltech +// All rights not granted herein are expressly reserved by Caltech. +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +package main + +import ( + "encoding/csv" + "flag" + "fmt" + "io" + "os" + "path" + "strings" + + // CaltechLibrary packages + "github.com/caltechlibrary/cli" + "github.com/caltechlibrary/datatools" + + // 3rd Party packages + "github.com/tealeg/xlsx" +) + +var ( + usage = `USAGE: %s [OPTIONS] EXCEL_WORKBOOK_NAME [SHEET_NAME]` + + description = ` +SYNOPSIS + +%s is a tool that converts individual Excel Sheets to CSV output. +` + + examples = ` +EXAMPLE + + %s my-workbook.xlsx "Sheet 1" > sheet1.csv + +This would get the first sheet from the workbook and save it as a CSV file. + + %s -c my-workbook.xlsx + +This will output the number of sheels in the Workbook. + + %s -n my-workbook.xlsx + +This will display a list of sheet names, one per line. +Putting it all together in a shell script. + + for SHEET_NAME in $(%s -n my-workbook.xlsx); do + %s my-workbook.xlsx "$SHEET_NAME" > \ + $(slugify "$SHEET_NAME").csv + done +` + + // Standard Options + showHelp bool + showLicense bool + showVersion bool + outputFName string + + // Application Options + showSheetCount bool + showSheetNames bool +) + +func sheetCount(workBookName string) (int, error) { + xlFile, err := xlsx.OpenFile(workBookName) + if err != nil { + return 0, err + } + return len(xlFile.Sheet), nil +} + +func sheetNames(workBookName string) ([]string, error) { + xlFile, err := xlsx.OpenFile(workBookName) + if err != nil { + return []string{}, err + } + result := []string{} + for sheetName, _ := range xlFile.Sheet { + result = append(result, sheetName) + } + return result, nil +} + +func xlsx2CSV(out io.Writer, workBookName, sheetName string) error { + xlFile, err := xlsx.OpenFile(workBookName) + if err != nil { + return err + } + results := [][]string{} + cells := []string{} + if sheet, ok := xlFile.Sheet[sheetName]; ok == true { + for _, row := range sheet.Rows { + cells = []string{} + for _, cell := range row.Cells { + val, err := cell.String() + if err != nil { + //val = fmt.Sprintf("%s", err) + } + cells = append(cells, val) + } + results = append(results, cells) + } + w := csv.NewWriter(out) + for _, record := range results { + if err := w.Write(record); err != nil { + return fmt.Errorf("error writing record to csv: %s", err) + } + } + w.Flush() + if err := w.Error(); err != nil { + return err + } + } + return fmt.Errorf("%s in worksheet %s", sheetName, workBookName) +} + +func init() { + // Standard Options + flag.BoolVar(&showHelp, "h", false, "display help") + flag.BoolVar(&showHelp, "help", false, "display help") + flag.BoolVar(&showLicense, "l", false, "display license") + flag.BoolVar(&showLicense, "license", false, "display license") + flag.BoolVar(&showVersion, "v", false, "display version") + flag.BoolVar(&showVersion, "version", false, "display version") + flag.StringVar(&outputFName, "o", "", "output filename") + flag.StringVar(&outputFName, "output", "", "output filename") + + // App Specific Options + flag.BoolVar(&showSheetCount, "c", false, "display number of sheets in Excel Workbook") + flag.BoolVar(&showSheetNames, "n", false, "display sheet names in Excel W9rkbook") +} + +func main() { + appName := path.Base(os.Args[0]) + flag.Parse() + args := flag.Args() + + // Configuration and command line interation + cfg := cli.New(appName, appName, fmt.Sprintf(datatools.LicenseText, appName, datatools.Version), datatools.Version) + cfg.UsageText = fmt.Sprintf(usage, appName) + cfg.DescriptionText = fmt.Sprintf(description, appName) + cfg.ExampleText = fmt.Sprintf(examples, appName, appName, appName, appName, appName) + + if showHelp == true { + fmt.Println(cfg.Usage()) + os.Exit(0) + } + + if showLicense == true { + fmt.Println(cfg.License()) + os.Exit(0) + } + + if showVersion == true { + fmt.Println(cfg.Version()) + os.Exit(0) + } + + out, err := cli.Create(outputFName, os.Stdout) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + defer cli.CloseFile(outputFName, out) + + if len(args) < 1 { + fmt.Println(cfg.Usage()) + fmt.Fprintln(os.Stderr, "Missing Excel Workbook names") + os.Exit(1) + } + + workBookName := args[0] + if showSheetCount == true { + if cnt, err := sheetCount(workBookName); err == nil { + fmt.Printf("%d", cnt) + os.Exit(0) + } else { + fmt.Fprintf(os.Stderr, "%s, %s\n", workBookName, err) + os.Exit(1) + } + } + + if showSheetNames == true { + if names, err := sheetNames(workBookName); err == nil { + fmt.Println(strings.Join(names, "\n")) + os.Exit(0) + } else { + fmt.Fprintf(os.Stderr, "%s, %s\n", workBookName, err) + os.Exit(1) + } + } + + if len(args) < 2 { + fmt.Fprintln(os.Stderr, "Missing worksheet name") + os.Exit(1) + } + for _, sheetName := range args[1:] { + if len(sheetName) > 0 { + xlsx2CSV(out, workBookName, sheetName) + } + } +} diff --git a/cmds/xlsx2json/xlsx2json.go b/cmds/xlsx2json/xlsx2json.go new file mode 100644 index 0000000..9abcc54 --- /dev/null +++ b/cmds/xlsx2json/xlsx2json.go @@ -0,0 +1,219 @@ +// +// xlsx2json.go is a command line utility that converts an Excel +// Workboom Sheet into JSON. +// +// @Author R. S. Doiel +// +// Copyright (c) 2017, Caltech +// All rights not granted herein are expressly reserved by Caltech. +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path" + "strings" + + // CaltechLibrary packages + "github.com/caltechlibrary/cli" + "github.com/caltechlibrary/datatools" + + // 3rd Party packages + "github.com/tealeg/xlsx" +) + +var ( + usage = `USAGE: %s [OPTIONS] EXCEL_WORKBOOK_NAME [SHEET_NAME]` + + description = ` +SYNOPSIS + +%s is a tool that converts individual Excel Workbook Sheets into +JSON output. +` + + examples = ` +EXAMPLE + + %s my-workbook.xlsx "Sheet 1" > sheet1.json + +This would get the first sheet from the workbook and save it as a JSON file. + + %s -c my-workbook.xlsx + +This will output the number of sheels in the Workbook. + + %s -n my-workbook.xlsx + +This will display a list of sheet names, one per line. +Putting it all together in a shell script. + + for SHEET_NAME in $(%s -n my-workbook.xlsx); do + %s my-workbook.xlsx "$SHEET_NAME" > \ + $(slugify "$SHEET_NAME").json + done +` + + // Standard Options + showHelp bool + showLicense bool + showVersion bool + outputFName string + + // Application Options + showSheetCount bool + showSheetNames bool +) + +func sheetCount(workBookName string) (int, error) { + xlFile, err := xlsx.OpenFile(workBookName) + if err != nil { + return 0, err + } + return len(xlFile.Sheet), nil +} + +func sheetNames(workBookName string) ([]string, error) { + xlFile, err := xlsx.OpenFile(workBookName) + if err != nil { + return []string{}, err + } + result := []string{} + for sheetName, _ := range xlFile.Sheet { + result = append(result, sheetName) + } + return result, nil +} + +func xlsx2JSON(out *os.File, workBookName, sheetName string) error { + xlFile, err := xlsx.OpenFile(workBookName) + if err != nil { + return err + } + results := [][]string{} + cells := []string{} + if sheet, ok := xlFile.Sheet[sheetName]; ok == true { + for _, row := range sheet.Rows { + cells = []string{} + for _, cell := range row.Cells { + val, err := cell.String() + if err != nil { + //val = fmt.Sprintf("%s", err) + } + cells = append(cells, val) + } + results = append(results, cells) + } + src, err := json.Marshal(results) + if err != nil { + return err + } + fmt.Fprintf(out, "%s", src) + return nil + } + return fmt.Errorf("%s is missing from worksheet %s", sheetName, workBookName) +} + +func init() { + // Standard Options + flag.BoolVar(&showHelp, "h", false, "display help") + flag.BoolVar(&showHelp, "help", false, "display help") + flag.BoolVar(&showLicense, "l", false, "display license") + flag.BoolVar(&showLicense, "license", false, "display license") + flag.BoolVar(&showVersion, "v", false, "display version") + flag.BoolVar(&showVersion, "version", false, "display version") + flag.StringVar(&outputFName, "o", "", "output filename") + flag.StringVar(&outputFName, "output", "", "output filename") + + // App Specific Options + flag.BoolVar(&showSheetCount, "c", false, "display number of sheets in Excel Workbook") + flag.BoolVar(&showSheetNames, "n", false, "display sheet names in Excel Workbook") +} + +func main() { + appName := path.Base(os.Args[0]) + flag.Parse() + args := flag.Args() + + // Configuration and command line interation + cfg := cli.New(appName, appName, fmt.Sprintf(datatools.LicenseText, appName, datatools.Version), datatools.Version) + cfg.UsageText = fmt.Sprintf(usage, appName) + cfg.DescriptionText = fmt.Sprintf(description, appName) + cfg.ExampleText = fmt.Sprintf(examples, appName, appName, appName, appName, appName) + + if showHelp == true { + fmt.Println(cfg.Usage()) + os.Exit(0) + } + + if showLicense == true { + fmt.Println(cfg.License()) + os.Exit(0) + } + + if showVersion == true { + fmt.Println(cfg.Version()) + os.Exit(0) + } + + out, err := cli.Create(outputFName, os.Stdout) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + defer cli.CloseFile(outputFName, out) + + if len(args) < 1 { + fmt.Println(cfg.Usage()) + fmt.Fprintln(os.Stderr, "Missing Excel Workbook names") + os.Exit(1) + } + + workBookName := args[0] + if showSheetCount == true { + if cnt, err := sheetCount(workBookName); err == nil { + fmt.Fprintf(out, "%d", cnt) + os.Exit(0) + } else { + fmt.Fprintf(os.Stderr, "%s, %s\n", workBookName, err) + os.Exit(1) + } + } + + if showSheetNames == true { + if names, err := sheetNames(workBookName); err == nil { + fmt.Fprintln(out, strings.Join(names, "\n")) + os.Exit(0) + } else { + fmt.Fprintf(os.Stderr, "%s, %s\n", workBookName, err) + os.Exit(1) + } + } + + if len(args) < 2 { + fmt.Fprintln(os.Stderr, "Missing worksheet name") + os.Exit(1) + } + for _, sheetName := range args[1:] { + if len(sheetName) > 0 { + err := xlsx2JSON(out, workBookName, sheetName) + if err != nil { + fmt.Fprintf(os.Stderr, "%s, %s, error: %s\n", workBookName, sheetName, err) + os.Exit(1) + } + } + } +} diff --git a/css/site.css b/css/site.css new file mode 100644 index 0000000..d5a4114 --- /dev/null +++ b/css/site.css @@ -0,0 +1,245 @@ +/** + * site.css - stylesheet for the Caltech Library's Digital Library Development Group's sandbox. + * + * orange: #FF6E1E; + * + * Secondary pallet: + * + * lightgrey: #C8C8C8 + * grey: #76777B + * darkgrey: #616265 + * slategrey: #AAA99F + * + * Impact Pallete see: http://identity.caltech.edu/web/colors + */ +body { + margin: 0; + border: 0; + padding: 0; + width: 100%; + height: 100%; + color: black; + background-color: #AAA99F; /* #76777B;*/ + /* + color: #FF6E1E; + background-color: white; + */ + font-family: Open Sans, Helvetica, Sans-Serif; + font-size: 16px; +} + +header { + position: relative; + display: block; + color: white; + background-color: white; + z-index: 2; + height: 105px; + vertical-align: middle; +} + +header img { + position: relative; + display: inline; + padding-left: 20px; + margin: 0; + height: 42px; + padding-top: 32px; +} + +header h1 { + position: relative; + display: inline-block; + margin: 0; + border: 0; + padding: 0; + font-size: 3em; + font-weight: normal; + vertical-align: 0.78em; + color: #FF6E1E; +} + +header a, header a:link, header a:visited, header a:active, header a:hover, header a:focus { + color: #FF6E1E; + background-color: inherit; +} + + +a, a:link, a:visited { + color: #76777B; + background-color: inherit; + text-decoration: none; +} + +a:active, a:hover, a:focus { + color: #FF6E1E; + font-weight: bolder; +} + +nav { + position: relative; + display: block; + width: 100%; + margin: 0; + padding: 0; + font-size: 0.78em; + vertical-align: middle; + color: black; + background-color: #AAA99F; /* #76777B;*/ + text-align: left; +} + +nav div { + display: inline-block; + /* padding-left: 10em; */ + margin-left: 10em; + margin-right: 0; +} + +nav a, nav a:link, nav a:visited, nav a:active { + color: white; + background-color: inherit; + text-decoration: none; +} + +nav a:hover, nav a:focus { + color: #FF6E1E; + background-color: inherit; + text-decoration: none; +} + + +nav div h2 { + position: relative; + display: block; + min-width: 20%; + margin: 0; + font-size: 1.24em; + font-style: normal; +} + +nav div > ul { + display: none; + padding-left: 0.24em; + text-align: left; +} + +nav ul { + display: inline-block; + padding-left: 0.24em; + list-style-type: none; + text-align: left; + text-decoration: none; +} + +nav ul li { + display: inline; + padding: 1em; +} + +section { + position: relative; + display: inline-block; + width: 100%; + min-height: 84%; + color: black; + background-color: white; + margin: 0; + padding-top 0; + padding-bottom: 2em; + padding-left: 1em; + padding-right: 0; +} + +section h1 { + font-size: 1.32em; +} + +section h2 { + font-size: 1.12em; + font-weight: italic; +} + +section h3 { + font-size: 1em; + text-transform: uppercase; +} + +section ul { + display: block; + list-style: inside; + list-style-type: square; + margin: 0; + padding-left: 1.24em; +} + +aside { + margin: 0; + border: 0; + padding-left: 1em; + position: relative; + display: inline-block; + text-align: right; +} + +aside h2 { + font-size: 1em; + text-transform: uppercase; +} + +aside h2 > a { + font-style: normal; +} + +aside ul { + margin: 0; + padding: 0; + display: block; + list-style-type: none; +} + +aside ul li { + font-size: 0.82em; +} + +aside ul > ul { + padding-left: 1em; + font-size: 0.72em; +} + +footer { + position: relative; + bottom: 0; + display: block; + width: 100%; + height: 2em; + color: white; + background-color: #616265; + + font-size: 80%; + text-align: left; + vertical-align: middle; + z-index: 2; +} + +footer h1, footer span, footer address { + position: relative; + display: inline-block; + margin: 0; + padding-left: 0.24em; + font-family: Open Sans, Helvetica, Sans-Serif; + font-size: 1em; +} + +footer h1 { + font-weight: normal; +} + +footer a, footer a:link, footer a:visited, footer a:active, footer a:focus, footer a:hover { + padding: 0; + display: inline; + margin: 0; + color: #FF6E1E; + text-decoration: none; +} + diff --git a/csv2mdtable.md b/csv2mdtable.md new file mode 100644 index 0000000..4238a05 --- /dev/null +++ b/csv2mdtable.md @@ -0,0 +1,37 @@ + +# USAGE + + csv2mdtable [OPTIONS] + +## SYNOPSIS + +csv2mdtable reads CSV from stdin and writes a Github Flavored Markdown +table to stdout. + +## OPTIONS + +``` + -h display help + -help display help + -i set input file + -l display license + -license display license + -o set output file + -v display version + -version display version +``` + +## EXAMPLES + +Convert data1.csv to data1.md using Unix pipes. + +``` + cat data1.csv | csv2mdtable > data1.md +``` + +Convert data1.csv to data1.md using options. + +``` + csv2mdtable -i data1.csv -o data1.md +``` + diff --git a/csv2xlsx.md b/csv2xlsx.md new file mode 100644 index 0000000..c1bb856 --- /dev/null +++ b/csv2xlsx.md @@ -0,0 +1,41 @@ + +# USAGE + + csv2xlsx [OPTIONS] WORKBOOK_NAME SHEET_NAME + +## SYNOPSIS + +csv2xlsx will take CSV input and create a new sheet in an Excel Workbook. +If the Workbook does not exist then it is created. + +## OPTIONS + +``` + -h display help + -help display help + -i input filename (CSV content) + -input input filename (CSV content) + -l display license + -license display license + -sheet Sheet name to create/replace + -v display version + -version display version + -workbook Workbook name +``` + +## EXAMPLE + +``` + csv2xlsx -i data.csv MyWorkbook.xlsx 'My worksheet' +``` + +This creates a new 'My worksheet' in the Excel Workbook +called 'MyWorkbook.xlsx' with the contents of data.csv. + +``` + cat data.csv | csv2xlsx MyWorkbook.xlsx 'My worksheet' +``` + +This does the same but the contents of data.csv are piped into +the workbook's sheet. + diff --git a/csvcols.md b/csvcols.md new file mode 100644 index 0000000..7aaeebc --- /dev/null +++ b/csvcols.md @@ -0,0 +1,48 @@ +# USAGE + + csvcols [OPTIONS] ARGS_AS_COLS + +## SYNOPSIS + +csvcols converts a set of command line args into columns output in CSV format. +It can also be used to filter input CSV and rendering only the column numbers +listed on the commandline. + +## OPTIONS + +``` + -d set delimiter for conversion + -delimiter set delimiter for conversion + -f filter CSV input for columns requested + -filter-columns filter CSV input for columns requested + -h display help + -help display help + -l display license + -license display license + -v display version + -version display version +``` + +## EXAMPLES + +Simple usage of building a CSV file one row at a time. + +``` + csvcols one two three > 3col.csv + csvcols 1 2 3 >> 3col.csv + cat 3col.csv +``` + +Example parsing a pipe delimited string into a CSV line + +``` + csvcols -d "|" "one|two|three" > 3col.csv + csvcols -delimiter "|" "1|2|3" >> 3col.csv + cat 3col.csv +``` + +Filter a 10 column CSV file for columns 0,3,5 (left most column is number zero) + +``` + cat 10col.csv | csvcols -f 0 3 5 > 3col.csv +``` diff --git a/csvjoin.md b/csvjoin.md new file mode 100644 index 0000000..3a1869c --- /dev/null +++ b/csvjoin.md @@ -0,0 +1,32 @@ + +# USAGE + + csvjoin [OPTIONS] CSV1 CSV2 COL1 COL2 + +## SYNOPSIS + +csvjoin outputs CSV content based on two CSV files with match column values. +Each CSV input file has a designated column to match on. The values are +compared as strings. + +## OPTIONS + +``` + -h display help + -help display help + -l display license + -license display license + -v display version + -version display version +``` + +## EXAMPLES + +Simple usage of building a merged CSV file from data1.csv +and data2.csv where column 1 in data1.csv matches the value in +column 3 of data2.csv. + +``` + csvjoin data1.csv data2.csv 1 3 > merged-data.csv +``` + diff --git a/datatools.go b/datatools.go new file mode 100644 index 0000000..71f8189 --- /dev/null +++ b/datatools.go @@ -0,0 +1,28 @@ +// +// datatools package is a collection of Go based command +// line tools for working with JSON content +// +// @Author R. S. Doiel, + +package datatools + +const ( + Version = "v0.0.1" + + LicenseText = ` +%s %s + +Copyright (c) 2017, Caltech +All rights not granted herein are expressly reserved by Caltech. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +` +) diff --git a/demo/demo1.json b/demo/demo1.json new file mode 100644 index 0000000..7200c26 --- /dev/null +++ b/demo/demo1.json @@ -0,0 +1 @@ +{"name": "Doe, Jane", "email":"jane.doe@example.org", "age": 42} diff --git a/demo/demo2.json b/demo/demo2.json new file mode 100644 index 0000000..e4f1936 --- /dev/null +++ b/demo/demo2.json @@ -0,0 +1 @@ +["one", 2, {"label":"three","value":3}] diff --git a/demo/file1.csv b/demo/file1.csv new file mode 100644 index 0000000..e2d4ef5 --- /dev/null +++ b/demo/file1.csv @@ -0,0 +1,7 @@ +1,Fred Zip +2,Frida Kahlo +3,Diego Rivera +4,Jack Flanders +5,Vincent Van Gough +6,Mojo Sam +7,Little Frieda diff --git a/demo/file2.csv b/demo/file2.csv new file mode 100644 index 0000000..2b0ca1c --- /dev/null +++ b/demo/file2.csv @@ -0,0 +1,7 @@ +Fred Zip,fred.zip@fictional.example.com +Frida Kahlo,frida.kahlo@arts.mexico.example.org +Diego Rivera,diego.rivera@arts.mexico.example.org +Jack Flanders,captian.jack@zbs.fictional.example.org +Vincent Van Gough,vvg@arts.europe.example.org +Mojo Sam,mojo.sam@zbs.fictional.example.org +Little Frieda,little.frieda@zbs.functional.example.org diff --git a/demo/schedule.xlsx b/demo/schedule.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..12e2bebfd195f140419765e5cf989bbd64797b16 GIT binary patch literal 5986 zcmai21z1$w+8shbq(f3#Qer?-VW^>F2mukKJBAiULdgLHiBTFPr4gj0yQP(ql5*%S zi3@)ByYTtn>;KO4%$)t4z1Nv{oxRt)_J$~;p_2fxu&@9GDIf*FwIIK^cDLm=buxNp zWpCm9_XUv4-Oe_tUqYpshu}s(u|tsMegWDo|Fo=0=|lh?&uH{$lgSf4E<5OJaA#fD z_KHn%-J(O4)2g@J{rUS%J9R4UaA52oY=mLdb2@hZisteso!P91G2D2c? zy#`ZT2|WwUcw`D#WG0s+bIP1=w~eRWYrFY0zGt9j_cU*S!CysGEomM1| zU9K614#V(zh$1EyZM-3C&qcoLC;$ND|6}AC)XB!#(#*`onfuqzFGk+PL!I*kZnuFN zy)j=Ktd@SPEx-E#cwc_0Fj8D;V)B!6`D9F?GST8V^NsEI({+Y731CXOhpo!nJHnWv zva*L;WkufZ4``UVt<=nqkT3=FP$ZM++P0S)tTK6Oa$9u;nUP=q{&YVgWpD+VVkLY! zyGW~DlS zWz#GRCTEnSV`uHFh_8Mcs(O4%^-D)1i&n+Z{*y&(ys?vS`{%c#C!HTJvS+@6FnGo@;IQ-NmU#WqWCV?PTYX z*j&$T_6;YDM--*qbfFZE)1MtR!>1P@yiZ^oixJ@J(tuCppGaKowJHnB1+~FvB5b{% z!VVuf*L+_55@RhXHUc!;InaCQqX_BFY3>qPT`+g#_pvHWV-VnNLaIv=^W6M~=RhpY z3|FR8ijZ@dYW3j!pdrtxIq1BetIhWEeGXHg0!6VHXlao2ao;O3f8M4IGsS*ei1r&i zXChKb@c_n0LK|HUrUGlOfHBtZ!}14$3CG6=NvI~L?1PUY&lz;{L9fir^R;3qy=Kkx z(_1O^!+UhJL-}E33B`(>Un$zK`P?#FtJ;!(h*GZxa~${&jyZX%>c!Qr9!N)ej|+rk zJLZL$KUy-NtpHc0sD$j+1!GGS$0p^0K%$wgf)%S36HtNPXVE@qTZ!7!L3>hX`)d^n z{i^3j!Y3y_CKl)Gux-j6=JWmVQ`8%ynAj&>O)1=n8QRBEo(DHLq-nDo>ajXrgCfqm zmnN#YArhV$O?TF%()!vAOWq1Zmm$Vhyh}*77VJD0$?@m#nA4GI^=fEFA-e}Ycuc57 zXoTWK;7!c}+tMQOYNlbUGH(5nW!mIaeQ@=7!KkI`Xn@OSNCv1&FY=64R@|5lRj!26 zCeUIC2S0w6!f%EX_mpPUOZGUNTu>52{+pi>;1?oP>3m$~oN~7)%xE)a;f^c?#8Y(5>^ebpW| zm6At_BT9cr8c^re35}9j2#Gs5P$!aOvS1L%lF)|`-jtRP<@!;+J=X%BTz8;(>yny6 zJ!0z@x-;|rA>isC1qkTG$@;EGlJ7fO)wsmkiJ!|gECQve)%;mym?YgA zw~l1r^8;L8Y}!>2-L?DUQr2epIlGU-_Xi9@OI4$FGrY!wuBb~*za63bSe(5!dd^y% zRM#3hIkG4qrTdHAG{q42|4?wp_)-7Z(&PFy5mz zNfSrlSmS~?%bdJ>Q8UW3&xhU8Bl42WrEv(`x7DaJCx>~|E-=ah)eF8QNkQ0wBPCXm zT%;GTr`Xef!9y-51VJdMv4eAdL(bF{7HF5q!BcRBuwUe6a)j009?ME_ z*H2d}N2{>MQ7u=x@N5ACj!k$qq0sIls!9*JLKsg&S1c7so%k~eNExnPDTW+-l8phM zNLH&X)PXRsMVRBM{mYaXD3^hZ-ppFZ8_wk5QtbuJjg!JV?J8kx+3x z@2o;*F0H%`$3`e zG(*S!$dKf!ncTrbcS8L|UB|?7fyI4G!vp{t*#1=K8UJ!V*Xq21lbP-HYR`4q4(n6I zZsH-pm?3r#;&~qZH2{8pO~&6olg3ak3*VN8X=`92AuMgCX`!djv)Oii9!k}hN?9#QY4V=dZkV%@NhADN6&v6O3bRqmT~2|6Zvno<<2HN{3v-y-}Xhr#$P z*;WNchLiO$P{Li4j9Sh%#F(S$J<488r!FUbX%+fc&g8RV{b|(@JQaQ1vE8|Vew2+* zI?O;E1e3$rcx`i(#cgT*;G&Q{;!6$+vR9NpUS!DrN3TQj7YELkMowm?8ZJ(k7Vy&f zd8rL3paBUyxexIbr)r1U85Y(<%1Hx^8_VIZffZ{3i&1>lBOiUBFki_$F>`YZbBpM3 zG4@*za7=5uTfl>4FI0XuRVp)Ub^qLG`9^*Bs6 zFPX0a(sc@5ur7%Dr`d=5Z{}S*Y|VaAzM$J3^<9AE)c3?sF?2O6ps`;WGd5l+Mq{kU zI7{PA!3Q%4xP18% z$Yw-Kp77z^7oP+4M5T8^(uj6hyfa-HGc+d;v8EVom`&ffdtIR{6o0 z)M$s1L4yKHPeJFkv){?uzexfVSHk+1;|4cjnVnXdWo)Om_7*Mh#`*i1sJmM@yKz8S zZLNjbDeTb4Epy+<*U4iMt@nN|dV)2XapxN%`B7ComNW7es1MJ?4dTgxw%FcsH`6F5 z06Gn^DIL%39**|3FGS2I7N)I^D&u6d`;1V)3r z$5?1uYt^C=w{%5ZBb>fvc~$sC09hIk-fDC{G9ELLkckeU*|3O)J-a4cCgm66KQ*UX=f z&~E>@YwiVXRtGkJHReJl@vC+U#uaLjTkH5f@x@L8R8*{e$@5k!Lrpo zo&7p(1(0|2l=jk1U-V`Dws@xe)omu+6?kh#K{g3!w&V}8fI(@5Y6oKAd zo!tI{+RcxVvE82pB$J)M=^O<2G2*{NZ^a~+oe$K40h3F{hl zaSqQxRW^5v5~LI>(v~(d*GP}gXu~QtiK;Wf34Y}w>_N4vVv`-nacei;*|R5Vv!P!) z4{}Ycxs*?BjPb}5`7~b4ByIGDr&qGqo?tZ9z4I)w zJ~^qH83C2*%_Uz!?9g$D6MikkP<)WN!^*&U8W1$@2T4VRTOO{M?=-4ecCY&na~oBb z3=@T`2w(|J(bK$wo6@rD2L%UC662Naa#ivcW;RDkV?;@@7!iz%Vx{UC`!^XRz`z~x z_j<>FxKq`bD+T>rUS#9yuB`y!Br#APmIbim{hJiy?^oT?jy}1*`SXsV6RT16zllIXqbug z9I=W-4rexZ7d}Z$Q%5RwN(kBcCHdB{J`eGW8Zh_A8bJ9M`G47tOCR$0EBIwR3L^Wg zn|W}8QVcf?T?`Ukc-pdu6e)JA%<$=K@8T)jnppX4@9&ULsI!^`CHtH1o{G({t$QU8 zbC}VX@bqi)vtkQW0dX*rtAkb^K<<%Ks{kWj29Ue0ec6v>MwB#rE9GM|eL&%kS}!wS zv63k?PJ=u>j#bd^d!N zQy;(0In2Y`5jUR6E5gRyJv#gho)a~9QjO|%9J#%+Wxy*KgYXDF>PVLJ27Ak-7X{C{NV)(~)MsytSI-CqGNs*0|C?oqdqJUd!x|}Q}tcI-s z*D^M4q0U%)Nl;}2k+I(kF_~xc53^}3D0rPLhq4$!-LW#z%4ll@#t~QF!MA!Uh*xa> z0IFyDEyK0tP1=Midx$%OYsx?yEvZ2;lOmJ{TZj3K2{Z+7yO-LK7v=L%r}mc}&48xT zm0Tpk`bR59e9_~H6x80u%-+R7)5F2cS^si1mT7dWpaF4CP|u9sa99sIiDS7=puJ|K zQxj`onzA3RA*n8^XQ{S}%nm_Ue;yKDY|*B|n`~c`rxn}T1__O%4#D;dZwvI`6f^Jh zd!R-r#WQ2vplci1;^M@gY)7pACdhIno=8_hvOubPIYSh??M_95kq@(Y1fE1|JD-QXHn^(nW)p9ud~Ru=krm~+m!AGA&Tjq!{67x=cZBQZ z;?kR5h5N-KyhzOXKTh@cq1VOkvYuSU`9<^Joct|qzYo5y1DASu6`q8@2LBIr{2k}| vBD++Lt4JpLhlcz<@cNv&T!U8;e9`&L8v4=kxr|Gv|Dt@A)36K8TbBKuJjnK;)_F0S?u1!gT;b z($U-A-xcZX^Y?|6M1Th(qetBk4`CqdVfIA3Y!rd0BeQbHFQfzLAS1~m%`i(D2@h0) zW=BKk>SuRs!>nhm_hP7y?2K&lS_Aqv=8eQ-cw~3N<~WzHg_L5PVlSnIt)p4ut;cNczwWJD|AFh}k2F*^&RC}Xe!W^9 zM&nu^=3;=Ivvcj-c%8Cvk}>wmnB)LD3+E(XSjwGK3{Vw>bZ*uxB4+!}1VFM!4>>5d7) z$ikk2W?dH8OV<8)D4J?-XS?D#zWrxnQy3{})*WHUm_cZ{l)M~+MA@-nSdT^`+s zF+-Y-tPayg-KMyzT0h4ARdIC^t*w|9xt{UoY8Ve)!6%Bh@pSD#le(KW)x&bB&bd?x zTCC2nIheRR)mErf$F51R-l|7U#G5*i%`bfs7w|kINaIJj+VRPzkjZ*xFuv_=o%hg^ZUoPssYujV?k(~V_ZSv;!oirz1H4HAqgC^tspjs}2 zXZ~!B?vT${dd(l~#HQQMpAae`G#o+tyQT~L9D_qmxAlf24!2%{_&dH^pQ;(cKt9DB zfQDR89*VjtyQCF~%;C1v$)!hd3$FBzJ&4bmYX11TI~Z>(M=w*(6{VmkmN`QPDeIi? zpkwM9_o@f6F(fy^z%Mmdgr#Uw1U+|0 z>+u+8whPuVC9z+bi5e8;e>Ny5Fg*3!7oX*)20kwK-f%}#UvHux5N!aSpsq(M#h__4 zb@KdDMnyQDSC^bIi0?MnSAeGf+mi;TJPIt~rJFM5(>RBy*ewqW#Ls z%$kjz_}_HcTpJ&9ilcD?i{a2>5V5w+JIF9ag_tJSfX0hmZk`+Ms>@D0dCQ&wG9hlL zDxtg;sLidwm%$5PvfcRmFLSPxysZ)s1`+9(V<+nC3FO#+OP}^H`o4h(_&)C<(^rPm z5SAY#yG^Uuvqf5zP^Oo{*>9f$1*?i-&xWj29iM~Wf37KJ9xc$}y};HHvYkE+%WL4G zNxaOFb*(D%m8ND)Mf2BDB&9)e;lc0p?oslOjP!Wu z?cOZ~SM~i_?P+7N2xg}-N{gnwrJ^&@R@V-&yfnxvV`XWpQQ2 zPdBk>_j$~94PTv>^jh(?h&K)Wp8qbBy|PhIk$Rq2No-Z6_Zdz8cU`&B#kuD#+vHa} zfguGw;Gv=BKd1!}L$ph^ZhJFjAGPi_eS4%h!z2dSR(57IrS;T{h^o?3)+HaaKcBnh zy8fx5MF#Bhtwu6e9aS}tnwsv#{GhluEU_N2AhU_-%BdoMQr4Pyd0Jkv+gnCxod-Bu zDyk;V2jpH02{0D95y20iwVj04UAY%T=e1|*;-4$y(_D{U;f}HtpuQ;TSL}GTQOcT6 z^@H_~IDhCQOV3a+(&==>4%_%D`}|17w7{g1)m}dLmDLc$Cyj=0Y8^p&e6E&3BGLR# zswe8i$MFv*8ur^9h-fD%B(zxKx06o)v(3awcR-=si7$9O#sC=sVbC(_Imi2jS;XU# z5IE7`lIVj1lQEEj6Q#M*nN@|9cMTxmV=$NbE^I>S0qPm5SD2PmdIIiHZxutDz7ge> zck%mfu=pAN%zUfNLjS}VvBZ%%qvxtTUD{u|j}4#eyub>}(9^9YF{@aYQ&15NQDY6{ zFvKhX3La7MX5Hz=4L-O4Pp@X|gP`BP>T?O$+zFrhK+Duv(7s`I@4gmi6}^J&*??8J zm0JQRhPEQoTtl*=hWWg58|5Vh+XXTK_bLTW_Yaku_aO$t5=}WEn3Supoam+zSH*PO z(vBZ?6Awdv+~~sUF07EOUDY!ZOh@0tdTo_#rzDl|u32~I*)Q_5DjLM|`Bd;S75+4S?Tf%1FKDp$$(06k^G%PL5!pQo;(%=-iXD^*%8!x! z&SN=$D*w}159_~gtwl-QZg>a{`k~zyJ73#$Ur1|yzdpx$Eu5YgAw*|@fVqaPZmb<2 zGXcZU*ht6qA1X6T%OThSF*r92(qkqEraDtAMMI8#j{bZODt`Qgp;Y1@QOB<>4Q?a} zrJ_Sl*mMfh;yJ{6XWI=6q(wtNJX4OS1v`^@u7Iw_%P9Aqh#M+e{xeGX+|C@*-Y)&Q0;;dlv{2Q_ zZYkv365X5cUT{G}(h7S?g0rdZmKOfxc~Fal2Jrhm>}};?wT%(fM?t8r2=5B2PTXu2 z&uqkD!j!Jm6H*lg-Bs)*aI(SfH`oVp{<88d1?rW_hA5J@k6z2&eT%qEv&L4#Rv!%e*;WG|>A4Z`v` z<}E1I1B)XK0{WQA6?Gc;wegCeHS@S8WJjdqN?WGH;=S1HqUqDY` zeyk0>pKrZBgGrXet=r*IMpH7aE$X&kc|2Chh)PfRC1;5eRL%zx#|}us0yxO{hqEp* zXOe`3JVqle9oxJ(Xeh5R2jf3KFKhY$Mz(GL&GopS&1 z|8U2nIEROf=o$wVf9w|zIePH$J42kog9@Pf*NKGclM!eE0LKWQs|5G@u>ZLK02QOS AMgRZ+ literal 0 HcmV?d00001 diff --git a/jsoncols.md b/jsoncols.md new file mode 100644 index 0000000..7191f7c --- /dev/null +++ b/jsoncols.md @@ -0,0 +1,69 @@ + +# USAGE + + jsoncols [OPTIONS] [EXPRESSION] [INPUT_FILENAME] [OUTPUT_FILENAME] + +## SYSNOPSIS + +jsoncols provides for both interactive exploration of JSON structures like jid +and command line scripting flexibility for data extraction into delimited +columns. This is helpful in flattening content extracted from JSON blobs. +The default delimiter for each value extracted is a comma. This can be +overridden with an option. + ++ EXPRESSION can be an empty stirng or dot notation for an object's path ++ INPUT_FILENAME is the filename to read or a dash "-" if you want to + explicity read from stdin + + if not provided then jsoncols reads from stdin ++ OUTPUT_FILENAME is the filename to write or a dash "-" if you want to + explicity write to stdout + + if not provided then jsoncols write to stdout + +## OPTIONS + +``` + -d set the delimiter for multi-field output + -h display help + -i read JSON from a file + -input read JSON from a file + -l display license + -m display output in monochrome + -r run interactively + -repl run interactively + -v display version +``` + +## EXAMPLES + +If myblob.json contained + +``` + {"name": "Doe, Jane", "email":"jane.doe@example.org", "age": 42} +``` + +Getting just the name could be done with + +``` + jsoncols .name myblob.json +``` + +This would yeild + +``` + "Doe, Jane" +``` + +Flipping .name and .age into pipe delimited columns is as +easy as listing each field in the expression inside a +space delimited string. + +``` + jsoncols -d\| ".name .age" myblob.json +``` + +This would yeild + +``` + "Doe, Jane"|42 +``` + diff --git a/jsonrange.md b/jsonrange.md new file mode 100644 index 0000000..11373c0 --- /dev/null +++ b/jsonrange.md @@ -0,0 +1,85 @@ + +# USAGE + + jsonrange [OPTIONS] JSON_EXPRESSION + +## SYSNOPSIS + +jsonrange turns either the JSON expression that is a map or array into delimited +elements suitable for processing in a "for" style loop in Bash. If the +JSON expression is an array then the elements of the array are returned else +if the expression is a map/object then the keys or attribute names are turned. + ++ EXPRESSION can be an empty string contains a JSON array or map. + +## OPTIONS + +``` + -d set delimiter for range output + -delimiter set delimiter for range output + -dotpath range on given dot path + -h display help + -i read JSON from file + -input read JSON from file + -l display license + -length return the number of keys or values + -limit limit the number of items output + -p range on given dot path + -v display version +``` + +## EXAMPLES + +Working with a map + +``` + jsonrange '{"name": "Doe, Jane", "email":"jane.doe@example.org", "age": 42}' +``` + +This would yield + +``` + name + email + age +``` + +Working with an array + +``` + jsonrange '["one", 2, {"label":"three","value":3}]' +``` + +would yield + +``` + one + 2 + {"label":"three","value":3} +``` + +Checking the length of a map or array + +``` + jsonrange -length '["one","two","three"]' +``` + +would yield + +``` + 3 +``` + +Limitting the number of items returned + +``` + jsonrange -limit 2 '[1,2,3,4,5]' +``` + +would yield + +``` + 1 + 2 +``` + diff --git a/mk-website.bash b/mk-website.bash new file mode 100755 index 0000000..f50cf45 --- /dev/null +++ b/mk-website.bash @@ -0,0 +1,53 @@ +#!/bin/bash + +PROJECT="datatools" + +function checkApp() { + APP_NAME=$(which $1) + if [ "$APP_NAME" = "" ] && [ ! -f "./bin/$1" ]; then + echo "Missing $APP_NAME" + exit 1 + fi +} + +function softwareCheck() { + for APP_NAME in $@; do + checkApp $APP_NAME + done +} + +function MakePage () { + nav="$1" + content="$2" + html="$3" + # Always use the latest compiled mkpage + APP=$(which mkpage) + if [ -f ./bin/mkpage ]; then + APP="./bin/mkpage" + fi + + echo "Rendering $html" + $APP \ + "title=text:$PROJECT -- a small collection of file and shell utilities" \ + "nav=$nav" \ + "content=$content" \ + "sitebuilt=text:Updated $(date)" \ + page.tmpl > $html + git add $html +} + +echo "Checking necessary software is installed" +softwareCheck mkpage +echo "Generating website index.html" +MakePage nav.md README.md index.html +echo "Generating install.html" +MakePage nav.md INSTALL.md install.html +echo "Generating license.html" +MakePage nav.md "markdown:$(cat LICENSE)" license.html + +# Generate the individual command docuumentation pages +for FNAME in csvcols csvjoin jsoncols jsonrange xlsx2json xlsx2csv csv2mdtable csv2xlsx; do + echo "Generating $FNAME.html" + MakePage nav.md "$FNAME.md" "$FNAME.html" +done + diff --git a/nav.md b/nav.md new file mode 100644 index 0000000..ef18f6e --- /dev/null +++ b/nav.md @@ -0,0 +1,14 @@ + ++ [home](/) ++ [README](./) ++ [LICENSE](license.html) ++ [INSTALL](install.html) ++ [csvcols](csvcols.html) ++ [csvjoin](csvjoin.html) ++ [csv2mdtable](csv2mdtable.html) ++ [jsoncols](jsoncols.html) ++ [jsonrange](jsonrange.html) ++ [xlsx2json](xlsx2json.html) ++ [xlsx2csv](xlsx2csv.html) ++ [Github](https://github.com/caltechlibrary/datatools) + diff --git a/page.tmpl b/page.tmpl new file mode 100644 index 0000000..8e14922 --- /dev/null +++ b/page.tmpl @@ -0,0 +1,29 @@ + + + + Caltech Library's Digital Library Development Sandbox + + + + +

+Caltech Library logo +
+ + +
+{{ .content }} +
+ + + + diff --git a/publish.bash b/publish.bash new file mode 100755 index 0000000..d51b936 --- /dev/null +++ b/publish.bash @@ -0,0 +1,30 @@ +#!/bin/bash +# + +WORKING_BRANCH=$(git branch | grep '* ' | cut -d\ -f 2) +if [ "$WORKING_BRANCH" = "gh-pages" ]; then + git commit -am "publishing to gh-pages branch" + git push origin gh-pages +else + echo "You're in $WORKING_BRANCH branch" + echo "You need to pull in changes to the gh-pages branch to publish" + read -p "Pull into gh-pages and publish? Y/N " YES_NO + if [ "$YES_NO" = "Y" ] || [ "$YES_NO" = "y" ]; then + echo "Committing and pushing to $WORKING_BRANCH" + git commit -am "commiting to $WORKING_BANCH"; + git push origin "$WORKING_BRANCH"; + echo "Changing branchs from $WORKING_BRANCH to gh-pages"; + git checkout gh-pages + echo "Merging changes from origin gh-pages" + git pull origin gh-pages + git commit -am "merging origin gh-pages" + echo "Pulling changes from $WORKING_BRANCH info gh-pages" + git pull origin "$WORKING_BRANCH" + echo "Merging changes from $WORKING_BRANCH" + git commit -am "merging $WORKING_BRANCH with gh-pages" + echo "Pushing changes up and publishing" + git push origin gh-pages + echo "Changing back to your working branch $WORKING_BRANCH" + git checkout "$WORKING_BRANCH" + fi +fi diff --git a/xlsx2csv.md b/xlsx2csv.md new file mode 100644 index 0000000..91065c0 --- /dev/null +++ b/xlsx2csv.md @@ -0,0 +1,47 @@ + +# USAGE + + xlsx2csv [OPTIONS] EXCEL_WORKBOOK_NAME [SHEET_NAME] + +## SYNOPSIS + +xlsx2csv is a tool that converts individual Excel Sheets to CSV output. + +## OPTIONS + +``` + -c display number of sheets in Excel Workbook + -h display help + -l display license + -n display sheet names in Excel W9rkbook + -v display version +``` + +## EXAMPLE + +``` + xlsx2csv my-workbook.xlsx "Sheet 1" > sheet1.csv +``` + +This would get the first sheet from the workbook and save it as a CSV file. + +``` + xlsx2csv -c my-workbook.xlsx +``` + +This will output the number of sheels in the Workbook. + +``` + xlsx2csv -n my-workbook.xlsx +``` + +This will display a list of sheet names, one per line. +Putting it all together in a shell script. + +``` + for SHEET_NAME in $(xlsx2csv -n my-workbook.xlsx); do + xlsx2csv my-workbook.xlsx "$SHEET_NAME" > \ + $(slugify "$SHEET_NAME").csv + done +``` + diff --git a/xlsx2json.md b/xlsx2json.md new file mode 100644 index 0000000..0f93ccf --- /dev/null +++ b/xlsx2json.md @@ -0,0 +1,48 @@ + +# USAGE + + xlsx2json [OPTIONS] EXCEL_WORKBOOK_NAME [SHEET_NAME] + +## SYNOPSIS + +xlsx2json is a tool that converts individual Excel Workbook Sheets into +JSON output. + +## OPTIONS + +``` + -c display number of sheets in Excel Workbook + -h display help + -l display license + -n display sheet names in Excel Workbook + -v display version +``` + +## EXAMPLE + +``` + xlsx2json my-workbook.xlsx "Sheet 1" > sheet1.json +``` + +This would get the first sheet from the workbook and save it as a JSON file. + +``` + xlsx2json -c my-workbook.xlsx +``` + +This will output the number of sheels in the Workbook. + +``` + xlsx2json -n my-workbook.xlsx +``` + +This will display a list of sheet names, one per line. +Putting it all together in a shell script. + +``` + for SHEET_NAME in $(xlsx2json -n my-workbook.xlsx); do + xlsx2json my-workbook.xlsx "$SHEET_NAME" > \ + $(slugify "$SHEET_NAME").json + done +``` +