diff --git a/README.md b/README.md index c3c190e..cecbb31 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/intenthq/anon)](https://goreportcard.com/report/github.com/intenthq/anon) [![License](https://img.shields.io/npm/l/express.svg)](https://github.com/intenthq/anon/LICENSE) ![GitHub release](https://img.shields.io/github/release/intenthq/anon.svg) -Anon is a tool for taking delimited files and anonymising or transforming columns until the output is useful for applications where sensitive information cannot be exposed. +Anon is a tool for taking delimited files and anonymising or transforming columns/fields until the output is useful for applications where sensitive information cannot be exposed. Currently this tools supports both CSV and JSON files (with one level of depth). ## Installation @@ -24,17 +24,20 @@ anon [--config ] Anon is designed to take input from `STDIN` and by default will output the anonymised file to `STDOUT`: ```sh -anon < some_file.csv > some_file_anonymised.csv +anon < some_file > some_file_anonymised ``` ### Configuration -In order to be useful, Anon needs to be told what you want to do to each column of the CSV. The config is defined as a JSON file (defaults to a file called `config.json` in the current directory): +In order to be useful, Anon needs to be told what you want to do to each column/field of the input. The config is defined as a JSON file (defaults to a file called `config.json` in the current directory): ```json5 { - "csv": { - "delimiter": "," + // Name of the format of the input file + // Currently supports "csv" and "json" + "formatName": { + // Options for the format you have picked go here. + // See the documentation for the format you choose below. }, // Optionally define a number of rows to randomly sample down to. // To do it, it will hash (using FNV-1 32 bits) the column with the ID @@ -44,15 +47,14 @@ In order to be useful, Anon needs to be told what you want to do to each column // Number used to mod the hash of the id and determine if the row // has to be included in the sample or not "mod": 30000 - // Specify in which a column a unique ID exists on which the sampling can - // be performed. Indices are 0 based, so this would sample on the first - // column. - "idColumn": 0 }, // An array of actions to take on each column - indices are 0 based, so index // 0 in this array corresponds to column 1, and so on. // - // There must be an action for every column in the CSV. + // If anonymising a CSV, there must be an action for every column in it. + // If anonymising a JSON, there must be an action for each field that needs to + // be anonymised. If there is no action defined for a specific field, this + // field value will be left untouched. "actions": [ { // The no-op, leaves the input unchanged. @@ -61,7 +63,10 @@ In order to be useful, Anon needs to be told what you want to do to each column { // Takes a UK format postcode (eg. W1W 8BE) and just keeps the outcode // (eg. W1W). - "name": "outcode" + "name": "outcode", + // what field in the json this action needs to be applied. If a field in + // the json doesn't have an action defined, then it will be left untouched. + "jsonField": "postcode" }, { // Hash (SHA1) the input. @@ -100,6 +105,36 @@ In order to be useful, Anon needs to be told what you want to do to each column } ``` +## Formats + +You can use CSV or JSON files as input. + +### CSV + +For a CSV file you will need a config like this: + +```json5 +"csv": { + "delimiter": ",", + // Specify in which column a unique ID exists on which the sampling can + // be performed. Indices are 0 based, so this would sample on the first + // column. + "idColumn": "0" +} +``` + +### JSON + +For a JSON file you will need to define config like this: + +```json5 +"json": { + // Specify in which field a unique ID exists on which the sampling can + // be performed. + "idField": "id" +} +``` + ## Contributing Any contribution will be welcome, please refer to our [contributing guidelines](CONTRIBUTING.md) for more information. diff --git a/anonymisations.go b/anonymisations.go index 7ff53f9..b47b131 100644 --- a/anonymisations.go +++ b/anonymisations.go @@ -32,6 +32,7 @@ type RangeConfig struct { type ActionConfig struct { Name string Salt *string + JsonField *string DateConfig DateConfig RangeConfig []RangeConfig } @@ -48,6 +49,21 @@ func anonymisations(configs *[]ActionConfig) ([]Anonymisation, error) { return res, nil } +// Returns a map of anonymisations according to the config, indexed by JsonField +func anonymisationsMap(configs *[]ActionConfig) (map[string]Anonymisation, error) { + var err error + res := make(map[string]Anonymisation) + for _, config := range *configs { + if config.JsonField == nil { + return nil, errors.New("You need to define a JsonField for each action configured.") + } + if res[*config.JsonField], err = config.create(); err != nil { + return nil, err + } + } + return res, nil +} + // Returns the configured salt or a random one // if it's not set. func (ac *ActionConfig) saltOrRandom() string { diff --git a/anonymisations_test.go b/anonymisations_test.go index 8cacad8..03fc30e 100644 --- a/anonymisations_test.go +++ b/anonymisations_test.go @@ -32,23 +32,74 @@ func assertAnonymisationFunction(t *testing.T, expected Anonymisation, actual An func TestAnonymisations(t *testing.T) { t.Run("a valid configuration", func(t *testing.T) { + f1, f2 := "f1", "f2" conf := &[]ActionConfig{ ActionConfig{ - Name: "nothing", + Name: "nothing", + JsonField: &f1, }, ActionConfig{ - Name: "hash", - Salt: &salt, + Name: "hash", + Salt: &salt, + JsonField: &f2, }, } - anons, err := anonymisations(conf) - assert.NoError(t, err) - assertAnonymisationFunction(t, identity, anons[0], "a") - assertAnonymisationFunction(t, hash(salt), anons[1], "a") + t.Run("anonymisations should return an array with each anonymisation created", func(t *testing.T) { + anons, err := anonymisations(conf) + assert.NoError(t, err) + assertAnonymisationFunction(t, identity, anons[0], "a") + assertAnonymisationFunction(t, hash(salt), anons[1], "a") + }) + t.Run("anonymisationsMap should return a map with each anonymisation created and indexed by field", func(t *testing.T) { + anons, err := anonymisationsMap(conf) + assert.NoError(t, err) + assertAnonymisationFunction(t, identity, anons[f1], "a") + assertAnonymisationFunction(t, hash(salt), anons[f2], "a") + }) }) t.Run("an invalid configuration", func(t *testing.T) { conf := &[]ActionConfig{ActionConfig{Name: "year", DateConfig: DateConfig{Format: "3333"}}} - anons, err := anonymisations(conf) + t.Run("anonymisations should return an error", func(t *testing.T) { + anons, err := anonymisations(conf) + assert.Error(t, err, "should return an error") + assert.Nil(t, anons) + }) + t.Run("anonymisationsMap should return an error", func(t *testing.T) { + anons, err := anonymisationsMap(conf) + assert.Error(t, err, "should return an error") + assert.Nil(t, anons) + }) + }) +} + +func TestAnonymisationsMap(t *testing.T) { + var f1, f2 = "f1", "f2" + t.Run("a valid configuration", func(t *testing.T) { + conf := &[]ActionConfig{ + ActionConfig{ + JsonField: &f1, + Name: "nothing", + }, + ActionConfig{ + JsonField: &f2, + Name: "hash", + Salt: &salt, + }, + } + anons, err := anonymisationsMap(conf) + assert.NoError(t, err) + assertAnonymisationFunction(t, identity, anons[f1], "a") + assertAnonymisationFunction(t, hash(salt), anons[f2], "a") + }) + t.Run("an action configuration without JsonField defined", func(t *testing.T) { + conf := &[]ActionConfig{ActionConfig{Name: "year"}} + anons, err := anonymisationsMap(conf) + assert.Error(t, err, "should return an error") + assert.Nil(t, anons) + }) + t.Run("an invalid action configuration", func(t *testing.T) { + conf := &[]ActionConfig{ActionConfig{JsonField: &f1, Name: "year", DateConfig: DateConfig{Format: "3333"}}} + anons, err := anonymisationsMap(conf) assert.Error(t, err, "should return an error") assert.Nil(t, anons) }) diff --git a/config.go b/config.go index 126241c..6eea4f4 100644 --- a/config.go +++ b/config.go @@ -8,28 +8,29 @@ import ( // CsvConfig stores the config to read and write the csv file type CsvConfig struct { Delimiter string + IDColumn uint32 +} + +// JsonConfig stores the config to read and write the json file +type JsonConfig struct { + IDField string } // SamplingConfig stores the config to know how to sample the file type SamplingConfig struct { - Mod uint32 - IDColumn uint32 + Mod uint32 } // Config stores all the configuration type Config struct { - Csv CsvConfig + Csv *CsvConfig + Json *JsonConfig Sampling SamplingConfig Actions []ActionConfig } -var defaultCsvConfig = CsvConfig{ - Delimiter: ",", -} - var defaultSamplingConfig = SamplingConfig{ - Mod: 1, - IDColumn: 0, + Mod: 1, } var defaultActionsConfig = []ActionConfig{} @@ -42,7 +43,6 @@ func loadConfig(filename string) (*Config, error) { } decoder := json.NewDecoder(file) conf := Config{ - Csv: defaultCsvConfig, Sampling: defaultSamplingConfig, Actions: defaultActionsConfig, } diff --git a/config_test.go b/config_test.go index e90e5b4..928500d 100644 --- a/config_test.go +++ b/config_test.go @@ -22,12 +22,8 @@ func TestLoadConfig(t *testing.T) { conf, err := loadConfig("config_defaults_test.json") require.NoError(t, err, "should return no error if the config can be loaded") assert.Equal(t, Config{ - Csv: CsvConfig{ - Delimiter: ",", - }, Sampling: SamplingConfig{ - Mod: 1, - IDColumn: 0, + Mod: 1, }, Actions: []ActionConfig{}, }, *conf, "should fill the config with the default values") @@ -39,12 +35,12 @@ func TestLoadConfig(t *testing.T) { conf, err := loadConfig("config_test.json") require.NoError(t, err, "should return no error if the config can be loaded") assert.Equal(t, Config{ - Csv: CsvConfig{ + Csv: &CsvConfig{ Delimiter: "|", + IDColumn: 84, }, Sampling: SamplingConfig{ - Mod: 77, - IDColumn: 84, + Mod: 77, }, Actions: []ActionConfig{ ActionConfig{ diff --git a/config_test.json b/config_test.json index cef3359..912e1eb 100644 --- a/config_test.json +++ b/config_test.json @@ -1,10 +1,10 @@ { "csv": { - "delimiter": "|" + "delimiter": "|", + "idColumn": 84 }, "sampling": { - "mod": 77, - "idColumn": 84 + "mod": 77 }, "actions": [ { diff --git a/csv_processor.go b/csv_processor.go new file mode 100644 index 0000000..35b912c --- /dev/null +++ b/csv_processor.go @@ -0,0 +1,70 @@ +package main + +import ( + "encoding/csv" + "fmt" + "io" + "log" + "os" +) + +func processCsv(inputFile string, outputFile string, conf *Config) error { + r := initCsvReader(inputFile, conf.Csv) + w := initCsvWriter(outputFile, conf.Csv) + + anons, err := anonymisations(&conf.Actions) + if err != nil { + return err + } + + if err := anonymiseCsv(r, w, conf, &anons); err != nil { + return err + } + + return nil +} + +func initCsvReader(filename string, conf *CsvConfig) *csv.Reader { + reader := csv.NewReader(fileOr(filename, os.Stdin, os.Open)) + reader.Comma = []rune(conf.Delimiter)[0] + return reader +} + +func initCsvWriter(filename string, conf *CsvConfig) *csv.Writer { + writer := csv.NewWriter(fileOr(filename, os.Stdout, os.Create)) + writer.Comma = []rune(conf.Delimiter)[0] + return writer +} + +func anonymiseCsv(r *csv.Reader, w *csv.Writer, conf *Config, anons *[]Anonymisation) error { + i := 0 + + for { + record, err := r.Read() + if err == io.EOF { + break + } else if pe, ok := err.(*csv.ParseError); ok && pe.Err == csv.ErrFieldCount { + // we just print the error and skip the record + log.Print(err) + } else if err != nil { + return err + } else if int64(conf.Csv.IDColumn) >= int64(len(record)) { + return fmt.Errorf("id column (%d) out of range, record has %d columns", conf.Csv.IDColumn, len(record)) + } else if sample(record[conf.Csv.IDColumn], conf.Sampling) { + anonymised, err := anonymise(record, *anons) + if err != nil { + // we just print the error and skip the record + log.Print(err) + } else { + w.Write(anonymised) + } + //TODO decide how often do we want to flush + if i%100 == 0 { + w.Flush() + } + } + i++ + } + w.Flush() + return nil +} diff --git a/csv_processor_test.go b/csv_processor_test.go new file mode 100644 index 0000000..7b9bf04 --- /dev/null +++ b/csv_processor_test.go @@ -0,0 +1,137 @@ +package main + +import ( + "bytes" + "encoding/csv" + "io/ioutil" + "log" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +var defaultCsvConfig = CsvConfig{ + Delimiter: ",", + IDColumn: 0, +} + +func TestAnonymiseCsv(t *testing.T) { + config := func(mod uint32, idColumn uint32) *Config { + return &Config{Csv: &CsvConfig{Delimiter: defaultCsvConfig.Delimiter, IDColumn: idColumn}, Sampling: SamplingConfig{Mod: mod}} + } + anons := &[]Anonymisation{identity, outcode} + createReaderAndWriter := func(in string) (*csv.Reader, *csv.Writer, *bytes.Buffer) { + var out bytes.Buffer + r := csv.NewReader(strings.NewReader(in)) + + w := csv.NewWriter(&out) + return r, w, &out + } + t.Run("when the id column is out of range", func(t *testing.T) { + r, w, out := createReaderAndWriter("a,b c\nd,e f\n") + + err := anonymiseCsv(r, w, config(1, 100), anons) + assert.Error(t, err, "should return an error") + assert.Equal(t, "", out.String(), "shouldn't write any output") + }) + t.Run("when there is an error writing the output", func(t *testing.T) { + var out bytes.Buffer + f, _ := os.Open("non existing file") + r := csv.NewReader(f) + + w := csv.NewWriter(&out) + err := anonymiseCsv(r, w, config(1, 0), anons) + assert.Error(t, err, "should return an error") + }) + t.Run("when there is an error processing one of the rows", func(t *testing.T) { + r, w, out := createReaderAndWriter("20020202\nfail\n10010101") + + y, _ := year("20060102") + err := anonymiseCsv(r, w, config(1, 0), &[]Anonymisation{y}) + assert.NoError(t, err, "should not return an error") + assert.Equal(t, "2002\n1001\n", out.String(), "should skip that row") + }) + t.Run("when sampling is defined", func(t *testing.T) { + r, w, out := createReaderAndWriter("a,b c\nd,e f\ng,h i\nj,k l\n") + + err := anonymiseCsv(r, w, config(2, 0), anons) + assert.NoError(t, err, "should return no error") + assert.Equal(t, "a,b\ng,h\n", out.String(), "should process some rows") + }) + t.Run("when all the rows are valid", func(t *testing.T) { + r, w, out := createReaderAndWriter("a,b c\nd,e f\n") + + err := anonymiseCsv(r, w, config(1, 0), anons) + assert.NoError(t, err, "should return no error") + assert.Equal(t, "a,b\nd,e\n", out.String(), "should process all rows") + }) +} + +func TestInitReader(t *testing.T) { + t.Run("with an empty filename", func(t *testing.T) { + tmpfile := tmpFile("content") + defer os.Remove(tmpfile.Name()) // clean up + + oldStdin := os.Stdin + defer func() { os.Stdin = oldStdin }() // Restore original Stdin + os.Stdin = tmpfile + + r := initCsvReader("", &defaultCsvConfig) + record, err := r.Read() + + assert.NoError(t, err, "should return no error") + assert.Equal(t, []string{"content"}, record, "should return a csv reader that reads from stdin") + }) + t.Run("with a valid filename", func(t *testing.T) { + tmpfile := tmpFile("content") + defer os.Remove(tmpfile.Name()) // clean up + + r := initCsvReader(tmpfile.Name(), &defaultCsvConfig) + record, err := r.Read() + + assert.NoError(t, err, "should return no error") + assert.Equal(t, []string{"content"}, record, "should return a csv reader that reads from the file") + }) +} + +func tmpFile(content string) *os.File { + tmpfile, err := ioutil.TempFile("", "anon-test") + if err != nil { + log.Fatal(err) + } + ioutil.WriteFile(tmpfile.Name(), []byte("content"), os.ModePerm) + return tmpfile +} + +func TestInitWriter(t *testing.T) { + t.Run("with an empty filename", func(t *testing.T) { + tmpfile := tmpFile("") + defer os.Remove(tmpfile.Name()) // clean up + + oldStdout := os.Stdout + defer func() { os.Stdout = oldStdout }() // Restore original Stdout + os.Stdout = tmpfile + + w := initCsvWriter("", &defaultCsvConfig) + err := w.Write([]string{"csv", "content"}) + w.Flush() + + content, _ := ioutil.ReadFile(tmpfile.Name()) + assert.NoError(t, err, "should return no error") + assert.Equal(t, "csv,content\n", string(content), "should return a csv writer that writes to stdout") + }) + t.Run("with a valid filename", func(t *testing.T) { + tmpfile := tmpFile("") + defer os.Remove(tmpfile.Name()) // clean up + + w := initCsvWriter(tmpfile.Name(), &defaultCsvConfig) + err := w.Write([]string{"csv", "content"}) + w.Flush() + + content, _ := ioutil.ReadFile(tmpfile.Name()) + assert.NoError(t, err, "should return no error") + assert.Equal(t, "csv,content\n", string(content), "should return a csv writer that writes to stdout") + }) +} diff --git a/json_processor.go b/json_processor.go new file mode 100644 index 0000000..73d7ebd --- /dev/null +++ b/json_processor.go @@ -0,0 +1,45 @@ +package main + +import ( + "encoding/json" + "io" + "log" + "os" +) + +func processJson(inputFile string, outputFile string, conf *Config) error { + r := json.NewDecoder(fileOr(inputFile, os.Stdin, os.Open)) + w := json.NewEncoder(fileOr(outputFile, os.Stdout, os.Create)) + + anonsMap, err := anonymisationsMap(&conf.Actions) + if err != nil { + return err + } + + return anonymiseJson(r, w, conf, anonsMap) +} + +func anonymiseJson(r *json.Decoder, w *json.Encoder, conf *Config, anonsMap map[string]Anonymisation) error { + var values map[string]string + for { + err := r.Decode(&values) + if err == io.EOF { + break + } else if err != nil { + return err + } else if sample(values[conf.Json.IDField], conf.Sampling) { + for key, value := range values { + if anon, exists := anonsMap[key]; exists { + anonValue, err := anon(value) + if err != nil { + log.Printf("Error applying anonymisation, field won't be anonymised. Error: %s\n", err) + } else { + values[key] = anonValue + } + } + } + w.Encode(values) + } + } + return nil +} diff --git a/json_processor_test.go b/json_processor_test.go new file mode 100644 index 0000000..8d2cf21 --- /dev/null +++ b/json_processor_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +var defaultJsonConfig = JsonConfig{ + IDField: "id", +} + +func TestAnonymiseJson(t *testing.T) { + config := func(mod uint32) *Config { + return &Config{Json: &defaultJsonConfig, Sampling: SamplingConfig{Mod: mod}} + } + createDecoderAndEncoder := func(in string) (*json.Decoder, *json.Encoder, *bytes.Buffer) { + var out bytes.Buffer + r := json.NewDecoder(strings.NewReader(in)) + w := json.NewEncoder(&out) + return r, w, &out + } + t.Run("when the json is not valid", func(t *testing.T) { + r, w, _ := createDecoderAndEncoder(`not a json`) + + err := anonymiseJson(r, w, config(1), map[string]Anonymisation{"id": hash("salt")}) + assert.Error(t, err, "should return an error") + }) + t.Run("when there is an error applying one of the anonymisations", func(t *testing.T) { + input := `{"id": "id", "date": "not a date"}` + r, w, out := createDecoderAndEncoder(input) + + y, _ := year("20060102") + err := anonymiseJson(r, w, config(1), map[string]Anonymisation{"id": identity, "date": y}) + assert.NoError(t, err, "should return no error") + assert.JSONEq(t, input, out.String(), "should leave the field with the error untouched") + }) + t.Run("when a field doesn't have an anonymisation defined", func(t *testing.T) { + r, w, out := createDecoderAndEncoder(`{"id": "id", "field": "don't touch it"}`) + + err := anonymiseJson(r, w, config(1), map[string]Anonymisation{"id": hash("salt")}) + assert.NoError(t, err, "should return no error") + assert.JSONEq(t, `{"id": "58619739af7a7374f30a027fe40313491e678ed9", "field": "don't touch it"}`, out.String(), "should leave the field with the error untouched") + }) + t.Run("when sampling is defined", func(t *testing.T) { + r, w, out := createDecoderAndEncoder(`{"id": "1"} + {"id": "2"} + {"id": "3"} + {"id": "4"}`) + + err := anonymiseJson(r, w, config(2), map[string]Anonymisation{"id": identity}) + assert.NoError(t, err, "should return no error") + assert.Equal(t, "{\"id\":\"1\"}\n{\"id\":\"3\"}\n", out.String(), "should process some rows") + }) + t.Run("when all the rows are valid", func(t *testing.T) { + r, w, out := createDecoderAndEncoder(`{"id": "1"} + {"id": "2"}`) + + err := anonymiseJson(r, w, config(1), map[string]Anonymisation{"id": identity}) + assert.NoError(t, err, "should return no error") + assert.Equal(t, "{\"id\":\"1\"}\n{\"id\":\"2\"}\n", out.String(), "should process all rows") + }) +} diff --git a/main.go b/main.go index 14154af..a17cf98 100644 --- a/main.go +++ b/main.go @@ -1,11 +1,8 @@ package main import ( - "encoding/csv" "flag" - "fmt" "hash/fnv" - "io" "log" "math/rand" "os" @@ -23,49 +20,21 @@ func main() { if err != nil { log.Fatal(err) } - r := initReader(flag.Arg(0), conf.Csv) - w := initWriter(*outputFile, conf.Csv) - anons, err := anonymisations(&conf.Actions) - if err != nil { - log.Fatal(err) - } - - if err := process(r, w, conf, &anons); err != nil { - log.Fatal(err) - } -} - -func process(r *csv.Reader, w *csv.Writer, conf *Config, anons *[]Anonymisation) error { - i := 0 - for { - record, err := r.Read() - if err == io.EOF { - break - } else if pe, ok := err.(*csv.ParseError); ok && pe.Err == csv.ErrFieldCount { - // we just print the error and skip the record - log.Print(err) - } else if err != nil { - return err - } else if int64(conf.Sampling.IDColumn) >= int64(len(record)) { - return fmt.Errorf("id column (%d) out of range, record has %d columns", conf.Sampling.IDColumn, len(record)) - } else if sample(record[conf.Sampling.IDColumn], conf.Sampling) { - anonymised, err := anonymise(record, *anons) - if err != nil { - // we just print the error and skip the record - log.Print(err) - } else { - w.Write(anonymised) - } - //TODO decide how often do we want to flush - if i%100 == 0 { - w.Flush() - } + if conf.Json != nil && conf.Csv != nil { + log.Fatal("Need to specify only one of Json or Csv.") + } else if &conf.Json != nil { + if err := processJson(flag.Arg(0), *outputFile, conf); err != nil { + log.Fatal(err) } - i++ + } else if &conf.Csv != nil { + if err := processCsv(flag.Arg(0), *outputFile, conf); err != nil { + log.Fatal(err) + } + } else { + log.Fatal("Need to specify at least one of Json or Csv.") } - w.Flush() - return nil + } func sample(s string, conf SamplingConfig) bool { @@ -74,18 +43,6 @@ func sample(s string, conf SamplingConfig) bool { return h.Sum32()%conf.Mod == 0 } -func initReader(filename string, conf CsvConfig) *csv.Reader { - reader := csv.NewReader(fileOr(filename, os.Stdin, os.Open)) - reader.Comma = []rune(conf.Delimiter)[0] - return reader -} - -func initWriter(filename string, conf CsvConfig) *csv.Writer { - writer := csv.NewWriter(fileOr(filename, os.Stdout, os.Create)) - writer.Comma = []rune(conf.Delimiter)[0] - return writer -} - // If filename is empty, will return `def`, if it's not, will return the // result of the function `action` after passing `filename` ot it. func fileOr(filename string, def *os.File, action func(string) (*os.File, error)) *os.File { diff --git a/main_test.go b/main_test.go index 4c17040..910e388 100644 --- a/main_test.go +++ b/main_test.go @@ -1,83 +1,13 @@ package main import ( - "bytes" - "encoding/csv" - "io/ioutil" - "log" + "errors" "os" - "strings" "testing" "github.com/stretchr/testify/assert" ) -func TestInitReader(t *testing.T) { - t.Run("with an empty filename", func(t *testing.T) { - tmpfile := tmpFile("content") - defer os.Remove(tmpfile.Name()) // clean up - - oldStdin := os.Stdin - defer func() { os.Stdin = oldStdin }() // Restore original Stdin - os.Stdin = tmpfile - - r := initReader("", defaultCsvConfig) - record, err := r.Read() - - assert.NoError(t, err, "should return no error") - assert.Equal(t, []string{"content"}, record, "should return a csv reader that reads from stdin") - }) - t.Run("with a valid filename", func(t *testing.T) { - tmpfile := tmpFile("content") - defer os.Remove(tmpfile.Name()) // clean up - - r := initReader(tmpfile.Name(), defaultCsvConfig) - record, err := r.Read() - - assert.NoError(t, err, "should return no error") - assert.Equal(t, []string{"content"}, record, "should return a csv reader that reads from the file") - }) -} - -func tmpFile(content string) *os.File { - tmpfile, err := ioutil.TempFile("", "anon-test") - if err != nil { - log.Fatal(err) - } - ioutil.WriteFile(tmpfile.Name(), []byte("content"), os.ModePerm) - return tmpfile -} - -func TestInitWriter(t *testing.T) { - t.Run("with an empty filename", func(t *testing.T) { - tmpfile := tmpFile("") - defer os.Remove(tmpfile.Name()) // clean up - - oldStdout := os.Stdout - defer func() { os.Stdout = oldStdout }() // Restore original Stdout - os.Stdout = tmpfile - - w := initWriter("", defaultCsvConfig) - err := w.Write([]string{"csv", "content"}) - w.Flush() - - content, _ := ioutil.ReadFile(tmpfile.Name()) - assert.NoError(t, err, "should return no error") - assert.Equal(t, "csv,content\n", string(content), "should return a csv writer that writes to stdout") - }) - t.Run("with a valid filename", func(t *testing.T) { - tmpfile := tmpFile("") - defer os.Remove(tmpfile.Name()) // clean up - - w := initWriter(tmpfile.Name(), defaultCsvConfig) - err := w.Write([]string{"csv", "content"}) - w.Flush() - - content, _ := ioutil.ReadFile(tmpfile.Name()) - assert.NoError(t, err, "should return no error") - assert.Equal(t, "csv,content\n", string(content), "should return a csv writer that writes to stdout") - }) -} func TestFileOr(t *testing.T) { assert.Equal(t, fileOr("", os.Stdin, stdOutOk), os.Stdin, "with an empty filename returns the default value") assert.Equal(t, fileOr("something", os.Stdin, stdOutOk), os.Stdout, "with non empty filename returns the value returned by the action") @@ -87,6 +17,10 @@ func stdOutOk(s string) (*os.File, error) { return os.Stdout, nil } +func stdOutErr(s string) (*os.File, error) { + return nil, errors.New("error") +} + func TestAnonymise(t *testing.T) { record := []string{"a", "b", "c"} actions := []Anonymisation{identity, hash(""), identity} @@ -103,55 +37,3 @@ func TestSample(t *testing.T) { assert.True(t, sample("a", conf)) assert.False(t, sample("b", conf)) } - -func TestProcess(t *testing.T) { - config := func(mod uint32, idColumn uint32) *Config { - return &Config{Sampling: SamplingConfig{Mod: mod, IDColumn: idColumn}} - } - anons := &[]Anonymisation{identity, outcode} - createReaderAndWriter := func(in string) (*csv.Reader, *csv.Writer, *bytes.Buffer) { - var out bytes.Buffer - r := csv.NewReader(strings.NewReader(in)) - - w := csv.NewWriter(&out) - return r, w, &out - } - t.Run("when the id column is out of range", func(t *testing.T) { - r, w, out := createReaderAndWriter("a,b c\nd,e f\n") - - err := process(r, w, config(1, 100), anons) - assert.Error(t, err, "should return an error") - assert.Equal(t, "", out.String(), "shouldn't write any output") - }) - t.Run("when there is an error writing the output", func(t *testing.T) { - var out bytes.Buffer - f, _ := os.Open("non existing file") - r := csv.NewReader(f) - - w := csv.NewWriter(&out) - err := process(r, w, config(1, 0), anons) - assert.Error(t, err, "should return an error") - }) - t.Run("when there is an error processing one of the rows", func(t *testing.T) { - r, w, out := createReaderAndWriter("20020202\nfail\n10010101") - - y, _ := year("20060102") - err := process(r, w, config(1, 0), &[]Anonymisation{y}) - assert.NoError(t, err, "should not return an error") - assert.Equal(t, "2002\n1001\n", out.String(), "should skip that row") - }) - t.Run("when sampling is defined", func(t *testing.T) { - r, w, out := createReaderAndWriter("a,b c\nd,e f\ng,h i\nj,k l\n") - - err := process(r, w, config(2, 0), anons) - assert.NoError(t, err, "should return no error") - assert.Equal(t, "a,b\ng,h\n", out.String(), "should process some rows") - }) - t.Run("when all the rows are valid", func(t *testing.T) { - r, w, out := createReaderAndWriter("a,b c\nd,e f\n") - - err := process(r, w, config(1, 0), anons) - assert.NoError(t, err, "should return no error") - assert.Equal(t, "a,b\nd,e\n", out.String(), "should process all rows") - }) -}