From 744ecb9136b6c1d0eb1c45a94274a9f407c01ddc Mon Sep 17 00:00:00 2001 From: Albert Pastrana Date: Wed, 2 Jan 2019 15:31:29 +0100 Subject: [PATCH] Add basic JSON support By applying this commit the `anon` tool will be able to read, anonymise and output basic JSON files. It currently only supports one level JSON fields. Some small refactor has been done and it's more than probable that the solution could be a bit DRYer, but don't have time to do so. Signed-off-by: Albert Pastrana --- README.md | 57 +++++++++++++---- anonymisations.go | 16 +++++ anonymisations_test.go | 67 +++++++++++++++++--- config.go | 20 +++--- config_test.go | 12 ++-- config_test.json | 6 +- csv_processor.go | 70 +++++++++++++++++++++ csv_processor_test.go | 137 +++++++++++++++++++++++++++++++++++++++++ json_processor.go | 45 ++++++++++++++ json_processor_test.go | 66 ++++++++++++++++++++ main.go | 67 ++++---------------- main_test.go | 128 ++------------------------------------ 12 files changed, 473 insertions(+), 218 deletions(-) create mode 100644 csv_processor.go create mode 100644 csv_processor_test.go create mode 100644 json_processor.go create mode 100644 json_processor_test.go diff --git a/README.md b/README.md index c3c190e..cecbb31 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/intenthq/anon)](https://goreportcard.com/report/github.com/intenthq/anon) [![License](https://img.shields.io/npm/l/express.svg)](https://github.com/intenthq/anon/LICENSE) ![GitHub release](https://img.shields.io/github/release/intenthq/anon.svg) -Anon is a tool for taking delimited files and anonymising or transforming columns until the output is useful for applications where sensitive information cannot be exposed. +Anon is a tool for taking delimited files and anonymising or transforming columns/fields until the output is useful for applications where sensitive information cannot be exposed. Currently this tools supports both CSV and JSON files (with one level of depth). ## Installation @@ -24,17 +24,20 @@ anon [--config ] Anon is designed to take input from `STDIN` and by default will output the anonymised file to `STDOUT`: ```sh -anon < some_file.csv > some_file_anonymised.csv +anon < some_file > some_file_anonymised ``` ### Configuration -In order to be useful, Anon needs to be told what you want to do to each column of the CSV. The config is defined as a JSON file (defaults to a file called `config.json` in the current directory): +In order to be useful, Anon needs to be told what you want to do to each column/field of the input. The config is defined as a JSON file (defaults to a file called `config.json` in the current directory): ```json5 { - "csv": { - "delimiter": "," + // Name of the format of the input file + // Currently supports "csv" and "json" + "formatName": { + // Options for the format you have picked go here. + // See the documentation for the format you choose below. }, // Optionally define a number of rows to randomly sample down to. // To do it, it will hash (using FNV-1 32 bits) the column with the ID @@ -44,15 +47,14 @@ In order to be useful, Anon needs to be told what you want to do to each column // Number used to mod the hash of the id and determine if the row // has to be included in the sample or not "mod": 30000 - // Specify in which a column a unique ID exists on which the sampling can - // be performed. Indices are 0 based, so this would sample on the first - // column. - "idColumn": 0 }, // An array of actions to take on each column - indices are 0 based, so index // 0 in this array corresponds to column 1, and so on. // - // There must be an action for every column in the CSV. + // If anonymising a CSV, there must be an action for every column in it. + // If anonymising a JSON, there must be an action for each field that needs to + // be anonymised. If there is no action defined for a specific field, this + // field value will be left untouched. "actions": [ { // The no-op, leaves the input unchanged. @@ -61,7 +63,10 @@ In order to be useful, Anon needs to be told what you want to do to each column { // Takes a UK format postcode (eg. W1W 8BE) and just keeps the outcode // (eg. W1W). - "name": "outcode" + "name": "outcode", + // what field in the json this action needs to be applied. If a field in + // the json doesn't have an action defined, then it will be left untouched. + "jsonField": "postcode" }, { // Hash (SHA1) the input. @@ -100,6 +105,36 @@ In order to be useful, Anon needs to be told what you want to do to each column } ``` +## Formats + +You can use CSV or JSON files as input. + +### CSV + +For a CSV file you will need a config like this: + +```json5 +"csv": { + "delimiter": ",", + // Specify in which column a unique ID exists on which the sampling can + // be performed. Indices are 0 based, so this would sample on the first + // column. + "idColumn": "0" +} +``` + +### JSON + +For a JSON file you will need to define config like this: + +```json5 +"json": { + // Specify in which field a unique ID exists on which the sampling can + // be performed. + "idField": "id" +} +``` + ## Contributing Any contribution will be welcome, please refer to our [contributing guidelines](CONTRIBUTING.md) for more information. diff --git a/anonymisations.go b/anonymisations.go index 7ff53f9..b47b131 100644 --- a/anonymisations.go +++ b/anonymisations.go @@ -32,6 +32,7 @@ type RangeConfig struct { type ActionConfig struct { Name string Salt *string + JsonField *string DateConfig DateConfig RangeConfig []RangeConfig } @@ -48,6 +49,21 @@ func anonymisations(configs *[]ActionConfig) ([]Anonymisation, error) { return res, nil } +// Returns a map of anonymisations according to the config, indexed by JsonField +func anonymisationsMap(configs *[]ActionConfig) (map[string]Anonymisation, error) { + var err error + res := make(map[string]Anonymisation) + for _, config := range *configs { + if config.JsonField == nil { + return nil, errors.New("You need to define a JsonField for each action configured.") + } + if res[*config.JsonField], err = config.create(); err != nil { + return nil, err + } + } + return res, nil +} + // Returns the configured salt or a random one // if it's not set. func (ac *ActionConfig) saltOrRandom() string { diff --git a/anonymisations_test.go b/anonymisations_test.go index 8cacad8..03fc30e 100644 --- a/anonymisations_test.go +++ b/anonymisations_test.go @@ -32,23 +32,74 @@ func assertAnonymisationFunction(t *testing.T, expected Anonymisation, actual An func TestAnonymisations(t *testing.T) { t.Run("a valid configuration", func(t *testing.T) { + f1, f2 := "f1", "f2" conf := &[]ActionConfig{ ActionConfig{ - Name: "nothing", + Name: "nothing", + JsonField: &f1, }, ActionConfig{ - Name: "hash", - Salt: &salt, + Name: "hash", + Salt: &salt, + JsonField: &f2, }, } - anons, err := anonymisations(conf) - assert.NoError(t, err) - assertAnonymisationFunction(t, identity, anons[0], "a") - assertAnonymisationFunction(t, hash(salt), anons[1], "a") + t.Run("anonymisations should return an array with each anonymisation created", func(t *testing.T) { + anons, err := anonymisations(conf) + assert.NoError(t, err) + assertAnonymisationFunction(t, identity, anons[0], "a") + assertAnonymisationFunction(t, hash(salt), anons[1], "a") + }) + t.Run("anonymisationsMap should return a map with each anonymisation created and indexed by field", func(t *testing.T) { + anons, err := anonymisationsMap(conf) + assert.NoError(t, err) + assertAnonymisationFunction(t, identity, anons[f1], "a") + assertAnonymisationFunction(t, hash(salt), anons[f2], "a") + }) }) t.Run("an invalid configuration", func(t *testing.T) { conf := &[]ActionConfig{ActionConfig{Name: "year", DateConfig: DateConfig{Format: "3333"}}} - anons, err := anonymisations(conf) + t.Run("anonymisations should return an error", func(t *testing.T) { + anons, err := anonymisations(conf) + assert.Error(t, err, "should return an error") + assert.Nil(t, anons) + }) + t.Run("anonymisationsMap should return an error", func(t *testing.T) { + anons, err := anonymisationsMap(conf) + assert.Error(t, err, "should return an error") + assert.Nil(t, anons) + }) + }) +} + +func TestAnonymisationsMap(t *testing.T) { + var f1, f2 = "f1", "f2" + t.Run("a valid configuration", func(t *testing.T) { + conf := &[]ActionConfig{ + ActionConfig{ + JsonField: &f1, + Name: "nothing", + }, + ActionConfig{ + JsonField: &f2, + Name: "hash", + Salt: &salt, + }, + } + anons, err := anonymisationsMap(conf) + assert.NoError(t, err) + assertAnonymisationFunction(t, identity, anons[f1], "a") + assertAnonymisationFunction(t, hash(salt), anons[f2], "a") + }) + t.Run("an action configuration without JsonField defined", func(t *testing.T) { + conf := &[]ActionConfig{ActionConfig{Name: "year"}} + anons, err := anonymisationsMap(conf) + assert.Error(t, err, "should return an error") + assert.Nil(t, anons) + }) + t.Run("an invalid action configuration", func(t *testing.T) { + conf := &[]ActionConfig{ActionConfig{JsonField: &f1, Name: "year", DateConfig: DateConfig{Format: "3333"}}} + anons, err := anonymisationsMap(conf) assert.Error(t, err, "should return an error") assert.Nil(t, anons) }) diff --git a/config.go b/config.go index 126241c..6eea4f4 100644 --- a/config.go +++ b/config.go @@ -8,28 +8,29 @@ import ( // CsvConfig stores the config to read and write the csv file type CsvConfig struct { Delimiter string + IDColumn uint32 +} + +// JsonConfig stores the config to read and write the json file +type JsonConfig struct { + IDField string } // SamplingConfig stores the config to know how to sample the file type SamplingConfig struct { - Mod uint32 - IDColumn uint32 + Mod uint32 } // Config stores all the configuration type Config struct { - Csv CsvConfig + Csv *CsvConfig + Json *JsonConfig Sampling SamplingConfig Actions []ActionConfig } -var defaultCsvConfig = CsvConfig{ - Delimiter: ",", -} - var defaultSamplingConfig = SamplingConfig{ - Mod: 1, - IDColumn: 0, + Mod: 1, } var defaultActionsConfig = []ActionConfig{} @@ -42,7 +43,6 @@ func loadConfig(filename string) (*Config, error) { } decoder := json.NewDecoder(file) conf := Config{ - Csv: defaultCsvConfig, Sampling: defaultSamplingConfig, Actions: defaultActionsConfig, } diff --git a/config_test.go b/config_test.go index e90e5b4..928500d 100644 --- a/config_test.go +++ b/config_test.go @@ -22,12 +22,8 @@ func TestLoadConfig(t *testing.T) { conf, err := loadConfig("config_defaults_test.json") require.NoError(t, err, "should return no error if the config can be loaded") assert.Equal(t, Config{ - Csv: CsvConfig{ - Delimiter: ",", - }, Sampling: SamplingConfig{ - Mod: 1, - IDColumn: 0, + Mod: 1, }, Actions: []ActionConfig{}, }, *conf, "should fill the config with the default values") @@ -39,12 +35,12 @@ func TestLoadConfig(t *testing.T) { conf, err := loadConfig("config_test.json") require.NoError(t, err, "should return no error if the config can be loaded") assert.Equal(t, Config{ - Csv: CsvConfig{ + Csv: &CsvConfig{ Delimiter: "|", + IDColumn: 84, }, Sampling: SamplingConfig{ - Mod: 77, - IDColumn: 84, + Mod: 77, }, Actions: []ActionConfig{ ActionConfig{ diff --git a/config_test.json b/config_test.json index cef3359..912e1eb 100644 --- a/config_test.json +++ b/config_test.json @@ -1,10 +1,10 @@ { "csv": { - "delimiter": "|" + "delimiter": "|", + "idColumn": 84 }, "sampling": { - "mod": 77, - "idColumn": 84 + "mod": 77 }, "actions": [ { diff --git a/csv_processor.go b/csv_processor.go new file mode 100644 index 0000000..35b912c --- /dev/null +++ b/csv_processor.go @@ -0,0 +1,70 @@ +package main + +import ( + "encoding/csv" + "fmt" + "io" + "log" + "os" +) + +func processCsv(inputFile string, outputFile string, conf *Config) error { + r := initCsvReader(inputFile, conf.Csv) + w := initCsvWriter(outputFile, conf.Csv) + + anons, err := anonymisations(&conf.Actions) + if err != nil { + return err + } + + if err := anonymiseCsv(r, w, conf, &anons); err != nil { + return err + } + + return nil +} + +func initCsvReader(filename string, conf *CsvConfig) *csv.Reader { + reader := csv.NewReader(fileOr(filename, os.Stdin, os.Open)) + reader.Comma = []rune(conf.Delimiter)[0] + return reader +} + +func initCsvWriter(filename string, conf *CsvConfig) *csv.Writer { + writer := csv.NewWriter(fileOr(filename, os.Stdout, os.Create)) + writer.Comma = []rune(conf.Delimiter)[0] + return writer +} + +func anonymiseCsv(r *csv.Reader, w *csv.Writer, conf *Config, anons *[]Anonymisation) error { + i := 0 + + for { + record, err := r.Read() + if err == io.EOF { + break + } else if pe, ok := err.(*csv.ParseError); ok && pe.Err == csv.ErrFieldCount { + // we just print the error and skip the record + log.Print(err) + } else if err != nil { + return err + } else if int64(conf.Csv.IDColumn) >= int64(len(record)) { + return fmt.Errorf("id column (%d) out of range, record has %d columns", conf.Csv.IDColumn, len(record)) + } else if sample(record[conf.Csv.IDColumn], conf.Sampling) { + anonymised, err := anonymise(record, *anons) + if err != nil { + // we just print the error and skip the record + log.Print(err) + } else { + w.Write(anonymised) + } + //TODO decide how often do we want to flush + if i%100 == 0 { + w.Flush() + } + } + i++ + } + w.Flush() + return nil +} diff --git a/csv_processor_test.go b/csv_processor_test.go new file mode 100644 index 0000000..7b9bf04 --- /dev/null +++ b/csv_processor_test.go @@ -0,0 +1,137 @@ +package main + +import ( + "bytes" + "encoding/csv" + "io/ioutil" + "log" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +var defaultCsvConfig = CsvConfig{ + Delimiter: ",", + IDColumn: 0, +} + +func TestAnonymiseCsv(t *testing.T) { + config := func(mod uint32, idColumn uint32) *Config { + return &Config{Csv: &CsvConfig{Delimiter: defaultCsvConfig.Delimiter, IDColumn: idColumn}, Sampling: SamplingConfig{Mod: mod}} + } + anons := &[]Anonymisation{identity, outcode} + createReaderAndWriter := func(in string) (*csv.Reader, *csv.Writer, *bytes.Buffer) { + var out bytes.Buffer + r := csv.NewReader(strings.NewReader(in)) + + w := csv.NewWriter(&out) + return r, w, &out + } + t.Run("when the id column is out of range", func(t *testing.T) { + r, w, out := createReaderAndWriter("a,b c\nd,e f\n") + + err := anonymiseCsv(r, w, config(1, 100), anons) + assert.Error(t, err, "should return an error") + assert.Equal(t, "", out.String(), "shouldn't write any output") + }) + t.Run("when there is an error writing the output", func(t *testing.T) { + var out bytes.Buffer + f, _ := os.Open("non existing file") + r := csv.NewReader(f) + + w := csv.NewWriter(&out) + err := anonymiseCsv(r, w, config(1, 0), anons) + assert.Error(t, err, "should return an error") + }) + t.Run("when there is an error processing one of the rows", func(t *testing.T) { + r, w, out := createReaderAndWriter("20020202\nfail\n10010101") + + y, _ := year("20060102") + err := anonymiseCsv(r, w, config(1, 0), &[]Anonymisation{y}) + assert.NoError(t, err, "should not return an error") + assert.Equal(t, "2002\n1001\n", out.String(), "should skip that row") + }) + t.Run("when sampling is defined", func(t *testing.T) { + r, w, out := createReaderAndWriter("a,b c\nd,e f\ng,h i\nj,k l\n") + + err := anonymiseCsv(r, w, config(2, 0), anons) + assert.NoError(t, err, "should return no error") + assert.Equal(t, "a,b\ng,h\n", out.String(), "should process some rows") + }) + t.Run("when all the rows are valid", func(t *testing.T) { + r, w, out := createReaderAndWriter("a,b c\nd,e f\n") + + err := anonymiseCsv(r, w, config(1, 0), anons) + assert.NoError(t, err, "should return no error") + assert.Equal(t, "a,b\nd,e\n", out.String(), "should process all rows") + }) +} + +func TestInitReader(t *testing.T) { + t.Run("with an empty filename", func(t *testing.T) { + tmpfile := tmpFile("content") + defer os.Remove(tmpfile.Name()) // clean up + + oldStdin := os.Stdin + defer func() { os.Stdin = oldStdin }() // Restore original Stdin + os.Stdin = tmpfile + + r := initCsvReader("", &defaultCsvConfig) + record, err := r.Read() + + assert.NoError(t, err, "should return no error") + assert.Equal(t, []string{"content"}, record, "should return a csv reader that reads from stdin") + }) + t.Run("with a valid filename", func(t *testing.T) { + tmpfile := tmpFile("content") + defer os.Remove(tmpfile.Name()) // clean up + + r := initCsvReader(tmpfile.Name(), &defaultCsvConfig) + record, err := r.Read() + + assert.NoError(t, err, "should return no error") + assert.Equal(t, []string{"content"}, record, "should return a csv reader that reads from the file") + }) +} + +func tmpFile(content string) *os.File { + tmpfile, err := ioutil.TempFile("", "anon-test") + if err != nil { + log.Fatal(err) + } + ioutil.WriteFile(tmpfile.Name(), []byte("content"), os.ModePerm) + return tmpfile +} + +func TestInitWriter(t *testing.T) { + t.Run("with an empty filename", func(t *testing.T) { + tmpfile := tmpFile("") + defer os.Remove(tmpfile.Name()) // clean up + + oldStdout := os.Stdout + defer func() { os.Stdout = oldStdout }() // Restore original Stdout + os.Stdout = tmpfile + + w := initCsvWriter("", &defaultCsvConfig) + err := w.Write([]string{"csv", "content"}) + w.Flush() + + content, _ := ioutil.ReadFile(tmpfile.Name()) + assert.NoError(t, err, "should return no error") + assert.Equal(t, "csv,content\n", string(content), "should return a csv writer that writes to stdout") + }) + t.Run("with a valid filename", func(t *testing.T) { + tmpfile := tmpFile("") + defer os.Remove(tmpfile.Name()) // clean up + + w := initCsvWriter(tmpfile.Name(), &defaultCsvConfig) + err := w.Write([]string{"csv", "content"}) + w.Flush() + + content, _ := ioutil.ReadFile(tmpfile.Name()) + assert.NoError(t, err, "should return no error") + assert.Equal(t, "csv,content\n", string(content), "should return a csv writer that writes to stdout") + }) +} diff --git a/json_processor.go b/json_processor.go new file mode 100644 index 0000000..73d7ebd --- /dev/null +++ b/json_processor.go @@ -0,0 +1,45 @@ +package main + +import ( + "encoding/json" + "io" + "log" + "os" +) + +func processJson(inputFile string, outputFile string, conf *Config) error { + r := json.NewDecoder(fileOr(inputFile, os.Stdin, os.Open)) + w := json.NewEncoder(fileOr(outputFile, os.Stdout, os.Create)) + + anonsMap, err := anonymisationsMap(&conf.Actions) + if err != nil { + return err + } + + return anonymiseJson(r, w, conf, anonsMap) +} + +func anonymiseJson(r *json.Decoder, w *json.Encoder, conf *Config, anonsMap map[string]Anonymisation) error { + var values map[string]string + for { + err := r.Decode(&values) + if err == io.EOF { + break + } else if err != nil { + return err + } else if sample(values[conf.Json.IDField], conf.Sampling) { + for key, value := range values { + if anon, exists := anonsMap[key]; exists { + anonValue, err := anon(value) + if err != nil { + log.Printf("Error applying anonymisation, field won't be anonymised. Error: %s\n", err) + } else { + values[key] = anonValue + } + } + } + w.Encode(values) + } + } + return nil +} diff --git a/json_processor_test.go b/json_processor_test.go new file mode 100644 index 0000000..8d2cf21 --- /dev/null +++ b/json_processor_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +var defaultJsonConfig = JsonConfig{ + IDField: "id", +} + +func TestAnonymiseJson(t *testing.T) { + config := func(mod uint32) *Config { + return &Config{Json: &defaultJsonConfig, Sampling: SamplingConfig{Mod: mod}} + } + createDecoderAndEncoder := func(in string) (*json.Decoder, *json.Encoder, *bytes.Buffer) { + var out bytes.Buffer + r := json.NewDecoder(strings.NewReader(in)) + w := json.NewEncoder(&out) + return r, w, &out + } + t.Run("when the json is not valid", func(t *testing.T) { + r, w, _ := createDecoderAndEncoder(`not a json`) + + err := anonymiseJson(r, w, config(1), map[string]Anonymisation{"id": hash("salt")}) + assert.Error(t, err, "should return an error") + }) + t.Run("when there is an error applying one of the anonymisations", func(t *testing.T) { + input := `{"id": "id", "date": "not a date"}` + r, w, out := createDecoderAndEncoder(input) + + y, _ := year("20060102") + err := anonymiseJson(r, w, config(1), map[string]Anonymisation{"id": identity, "date": y}) + assert.NoError(t, err, "should return no error") + assert.JSONEq(t, input, out.String(), "should leave the field with the error untouched") + }) + t.Run("when a field doesn't have an anonymisation defined", func(t *testing.T) { + r, w, out := createDecoderAndEncoder(`{"id": "id", "field": "don't touch it"}`) + + err := anonymiseJson(r, w, config(1), map[string]Anonymisation{"id": hash("salt")}) + assert.NoError(t, err, "should return no error") + assert.JSONEq(t, `{"id": "58619739af7a7374f30a027fe40313491e678ed9", "field": "don't touch it"}`, out.String(), "should leave the field with the error untouched") + }) + t.Run("when sampling is defined", func(t *testing.T) { + r, w, out := createDecoderAndEncoder(`{"id": "1"} + {"id": "2"} + {"id": "3"} + {"id": "4"}`) + + err := anonymiseJson(r, w, config(2), map[string]Anonymisation{"id": identity}) + assert.NoError(t, err, "should return no error") + assert.Equal(t, "{\"id\":\"1\"}\n{\"id\":\"3\"}\n", out.String(), "should process some rows") + }) + t.Run("when all the rows are valid", func(t *testing.T) { + r, w, out := createDecoderAndEncoder(`{"id": "1"} + {"id": "2"}`) + + err := anonymiseJson(r, w, config(1), map[string]Anonymisation{"id": identity}) + assert.NoError(t, err, "should return no error") + assert.Equal(t, "{\"id\":\"1\"}\n{\"id\":\"2\"}\n", out.String(), "should process all rows") + }) +} diff --git a/main.go b/main.go index 14154af..a17cf98 100644 --- a/main.go +++ b/main.go @@ -1,11 +1,8 @@ package main import ( - "encoding/csv" "flag" - "fmt" "hash/fnv" - "io" "log" "math/rand" "os" @@ -23,49 +20,21 @@ func main() { if err != nil { log.Fatal(err) } - r := initReader(flag.Arg(0), conf.Csv) - w := initWriter(*outputFile, conf.Csv) - anons, err := anonymisations(&conf.Actions) - if err != nil { - log.Fatal(err) - } - - if err := process(r, w, conf, &anons); err != nil { - log.Fatal(err) - } -} - -func process(r *csv.Reader, w *csv.Writer, conf *Config, anons *[]Anonymisation) error { - i := 0 - for { - record, err := r.Read() - if err == io.EOF { - break - } else if pe, ok := err.(*csv.ParseError); ok && pe.Err == csv.ErrFieldCount { - // we just print the error and skip the record - log.Print(err) - } else if err != nil { - return err - } else if int64(conf.Sampling.IDColumn) >= int64(len(record)) { - return fmt.Errorf("id column (%d) out of range, record has %d columns", conf.Sampling.IDColumn, len(record)) - } else if sample(record[conf.Sampling.IDColumn], conf.Sampling) { - anonymised, err := anonymise(record, *anons) - if err != nil { - // we just print the error and skip the record - log.Print(err) - } else { - w.Write(anonymised) - } - //TODO decide how often do we want to flush - if i%100 == 0 { - w.Flush() - } + if conf.Json != nil && conf.Csv != nil { + log.Fatal("Need to specify only one of Json or Csv.") + } else if &conf.Json != nil { + if err := processJson(flag.Arg(0), *outputFile, conf); err != nil { + log.Fatal(err) } - i++ + } else if &conf.Csv != nil { + if err := processCsv(flag.Arg(0), *outputFile, conf); err != nil { + log.Fatal(err) + } + } else { + log.Fatal("Need to specify at least one of Json or Csv.") } - w.Flush() - return nil + } func sample(s string, conf SamplingConfig) bool { @@ -74,18 +43,6 @@ func sample(s string, conf SamplingConfig) bool { return h.Sum32()%conf.Mod == 0 } -func initReader(filename string, conf CsvConfig) *csv.Reader { - reader := csv.NewReader(fileOr(filename, os.Stdin, os.Open)) - reader.Comma = []rune(conf.Delimiter)[0] - return reader -} - -func initWriter(filename string, conf CsvConfig) *csv.Writer { - writer := csv.NewWriter(fileOr(filename, os.Stdout, os.Create)) - writer.Comma = []rune(conf.Delimiter)[0] - return writer -} - // If filename is empty, will return `def`, if it's not, will return the // result of the function `action` after passing `filename` ot it. func fileOr(filename string, def *os.File, action func(string) (*os.File, error)) *os.File { diff --git a/main_test.go b/main_test.go index 4c17040..910e388 100644 --- a/main_test.go +++ b/main_test.go @@ -1,83 +1,13 @@ package main import ( - "bytes" - "encoding/csv" - "io/ioutil" - "log" + "errors" "os" - "strings" "testing" "github.com/stretchr/testify/assert" ) -func TestInitReader(t *testing.T) { - t.Run("with an empty filename", func(t *testing.T) { - tmpfile := tmpFile("content") - defer os.Remove(tmpfile.Name()) // clean up - - oldStdin := os.Stdin - defer func() { os.Stdin = oldStdin }() // Restore original Stdin - os.Stdin = tmpfile - - r := initReader("", defaultCsvConfig) - record, err := r.Read() - - assert.NoError(t, err, "should return no error") - assert.Equal(t, []string{"content"}, record, "should return a csv reader that reads from stdin") - }) - t.Run("with a valid filename", func(t *testing.T) { - tmpfile := tmpFile("content") - defer os.Remove(tmpfile.Name()) // clean up - - r := initReader(tmpfile.Name(), defaultCsvConfig) - record, err := r.Read() - - assert.NoError(t, err, "should return no error") - assert.Equal(t, []string{"content"}, record, "should return a csv reader that reads from the file") - }) -} - -func tmpFile(content string) *os.File { - tmpfile, err := ioutil.TempFile("", "anon-test") - if err != nil { - log.Fatal(err) - } - ioutil.WriteFile(tmpfile.Name(), []byte("content"), os.ModePerm) - return tmpfile -} - -func TestInitWriter(t *testing.T) { - t.Run("with an empty filename", func(t *testing.T) { - tmpfile := tmpFile("") - defer os.Remove(tmpfile.Name()) // clean up - - oldStdout := os.Stdout - defer func() { os.Stdout = oldStdout }() // Restore original Stdout - os.Stdout = tmpfile - - w := initWriter("", defaultCsvConfig) - err := w.Write([]string{"csv", "content"}) - w.Flush() - - content, _ := ioutil.ReadFile(tmpfile.Name()) - assert.NoError(t, err, "should return no error") - assert.Equal(t, "csv,content\n", string(content), "should return a csv writer that writes to stdout") - }) - t.Run("with a valid filename", func(t *testing.T) { - tmpfile := tmpFile("") - defer os.Remove(tmpfile.Name()) // clean up - - w := initWriter(tmpfile.Name(), defaultCsvConfig) - err := w.Write([]string{"csv", "content"}) - w.Flush() - - content, _ := ioutil.ReadFile(tmpfile.Name()) - assert.NoError(t, err, "should return no error") - assert.Equal(t, "csv,content\n", string(content), "should return a csv writer that writes to stdout") - }) -} func TestFileOr(t *testing.T) { assert.Equal(t, fileOr("", os.Stdin, stdOutOk), os.Stdin, "with an empty filename returns the default value") assert.Equal(t, fileOr("something", os.Stdin, stdOutOk), os.Stdout, "with non empty filename returns the value returned by the action") @@ -87,6 +17,10 @@ func stdOutOk(s string) (*os.File, error) { return os.Stdout, nil } +func stdOutErr(s string) (*os.File, error) { + return nil, errors.New("error") +} + func TestAnonymise(t *testing.T) { record := []string{"a", "b", "c"} actions := []Anonymisation{identity, hash(""), identity} @@ -103,55 +37,3 @@ func TestSample(t *testing.T) { assert.True(t, sample("a", conf)) assert.False(t, sample("b", conf)) } - -func TestProcess(t *testing.T) { - config := func(mod uint32, idColumn uint32) *Config { - return &Config{Sampling: SamplingConfig{Mod: mod, IDColumn: idColumn}} - } - anons := &[]Anonymisation{identity, outcode} - createReaderAndWriter := func(in string) (*csv.Reader, *csv.Writer, *bytes.Buffer) { - var out bytes.Buffer - r := csv.NewReader(strings.NewReader(in)) - - w := csv.NewWriter(&out) - return r, w, &out - } - t.Run("when the id column is out of range", func(t *testing.T) { - r, w, out := createReaderAndWriter("a,b c\nd,e f\n") - - err := process(r, w, config(1, 100), anons) - assert.Error(t, err, "should return an error") - assert.Equal(t, "", out.String(), "shouldn't write any output") - }) - t.Run("when there is an error writing the output", func(t *testing.T) { - var out bytes.Buffer - f, _ := os.Open("non existing file") - r := csv.NewReader(f) - - w := csv.NewWriter(&out) - err := process(r, w, config(1, 0), anons) - assert.Error(t, err, "should return an error") - }) - t.Run("when there is an error processing one of the rows", func(t *testing.T) { - r, w, out := createReaderAndWriter("20020202\nfail\n10010101") - - y, _ := year("20060102") - err := process(r, w, config(1, 0), &[]Anonymisation{y}) - assert.NoError(t, err, "should not return an error") - assert.Equal(t, "2002\n1001\n", out.String(), "should skip that row") - }) - t.Run("when sampling is defined", func(t *testing.T) { - r, w, out := createReaderAndWriter("a,b c\nd,e f\ng,h i\nj,k l\n") - - err := process(r, w, config(2, 0), anons) - assert.NoError(t, err, "should return no error") - assert.Equal(t, "a,b\ng,h\n", out.String(), "should process some rows") - }) - t.Run("when all the rows are valid", func(t *testing.T) { - r, w, out := createReaderAndWriter("a,b c\nd,e f\n") - - err := process(r, w, config(1, 0), anons) - assert.NoError(t, err, "should return no error") - assert.Equal(t, "a,b\nd,e\n", out.String(), "should process all rows") - }) -}