Merge pull request #67 from keboola/fix/separator-delimiter-from-manifest

Fix/separator delimiter from manifest

Author: kudj

docs/dao.html

@@ -2151,6 +2151,15 @@ <h2 id="args">Args</h2>
         self.delete_where_column = None
         self.delete_where_operator = None
 
+        if kwargs.get(&#39;delete_where_values&#39;):
+            self.delete_where_values = kwargs[&#39;delete_where_values&#39;]
+
+        if kwargs.get(&#39;delete_where_column&#39;):
+            self.delete_where_column = kwargs[&#39;delete_where_column&#39;]
+
+        if kwargs.get(&#39;delete_where_operator&#39;):
+            self.delete_where_operator = kwargs[&#39;delete_where_operator&#39;]
+
         self.set_delete_where_from_dict(delete_where)
         self.write_always = write_always
 
@@ -2185,7 +2194,8 @@ <h2 id="args">Args</h2>
                                 delimiter: Optional[str] = &#39;,&#39;,
                                 delete_where: Optional[dict] = None,
                                 write_always: Optional[bool] = False,
-                                schema: Optional[List[ColumnDefinition]] = None,
+                                schema: Optional[Union[TypeOrderedDict[str, ColumnDefinition], list[str]]] = None,
+                                **kwargs
                                 ):
         &#34;&#34;&#34;
         Factory method for creating a TableDefinition instance for output tables.
@@ -2220,6 +2230,7 @@ <h2 id="args">Args</h2>
                    delete_where=delete_where,
                    write_always=write_always,
                    schema=schema,
+                   **kwargs
                    )
 
     @classmethod
@@ -2237,7 +2248,7 @@ <h2 id="args">Args</h2>
                                delete_where: Optional[dict] = None,
                                stage: Optional[str] = &#39;in&#39;,
                                write_always: Optional[bool] = False,
-                               schema: Optional[List[ColumnDefinition]] = None,
+                               schema: Optional[Union[TypeOrderedDict[str, ColumnDefinition], list[str]]] = None,
                                rows_count: Optional[int] = None,
                                data_size_bytes: Optional[int] = None,
                                is_alias: Optional[bool] = False,
@@ -2247,7 +2258,9 @@ <h2 id="args">Args</h2>
                                id: Optional[str] = &#39;&#39;,
                                created: Optional[str] = None,
                                last_change_date: Optional[str] = None,
-                               last_import_date: Optional[str] = None):
+                               last_import_date: Optional[str] = None,
+                               **kwargs
+                               ):
         &#34;&#34;&#34;
         Factory method for creating a TableDefinition instance for input tables.
 
@@ -2302,7 +2315,9 @@ <h2 id="args">Args</h2>
                    id=id,
                    created=created,
                    last_change_date=last_change_date,
-                   last_import_date=last_import_date)
+                   last_import_date=last_import_date,
+                   **kwargs
+                   )
 
     @classmethod
     def convert_to_column_definition(cls, column_name, column_metadata, primary_key=False):
@@ -2390,6 +2405,9 @@ <h2 id="args">Args</h2>
         else:
             name = Path(manifest_file_path).stem
 
+        if manifest.get(&#39;name&#39;):
+            name = manifest.get(&#39;name&#39;)
+
         # test if the manifest is output and incompatible
         force_legacy_mode = False
         if not manifest.get(&#39;columns&#39;) and manifest.get(&#39;primary_key&#39;):
@@ -2399,28 +2417,57 @@ <h2 id="args">Args</h2>
 
         if manifest.get(&#39;id&#39;):
             stage = &#39;in&#39;
+            table_def = cls.build_input_definition(
+                # helper parameters
+                stage=stage,
+                force_legacy_mode=force_legacy_mode,
+                is_sliced=is_sliced,
+                full_path=full_path,
+
+                # basic in manifest parameters
+                id=manifest.get(&#39;id&#39;),
+                uri=manifest.get(&#39;uri&#39;),
+                name=name,
+                primary_key=manifest.get(&#39;primary_key&#39;),
+                created=manifest.get(&#39;created&#39;),
+                last_change_date=manifest.get(&#39;last_change_date&#39;),
+                last_import_date=manifest.get(&#39;last_import_date&#39;),
+                schema=cls.return_schema_from_manifest(manifest),
+                table_metadata=TableMetadata(manifest),
+
+                # additional in manifest parameters
+                rows_count=manifest.get(&#39;rows_count&#39;),
+                data_size_bytes=manifest.get(&#39;data_size_bytes&#39;),
+                is_alias=manifest.get(&#39;is_alias&#39;),
+                attributes=manifest.get(&#39;attributes&#39;),
+                indexed_columns=manifest.get(&#39;indexed_columns&#39;),
+            )
+
         else:
             stage = &#39;out&#39;
-
-        table_def = cls(name=name,
-                        stage=stage,
-                        full_path=full_path,
-                        is_sliced=is_sliced,
-                        id=manifest.get(&#39;id&#39;),
-                        table_metadata=TableMetadata(manifest),
-                        primary_key=manifest.get(&#39;primary_key&#39;),
-                        schema=cls.return_schema_from_manifest(manifest),
-                        uri=manifest.get(&#39;uri&#39;),
-                        created=manifest.get(&#39;created&#39;),
-                        last_change_date=manifest.get(&#39;last_change_date&#39;),
-                        last_import_date=manifest.get(&#39;last_import_date&#39;),
-                        rows_count=manifest.get(&#39;rows_count&#39;),
-                        data_size_bytes=manifest.get(&#39;data_size_bytes&#39;),
-                        is_alias=manifest.get(&#39;is_alias&#39;),
-                        force_legacy_mode=force_legacy_mode,
-                        indexed_columns=manifest.get(&#39;indexed_columns&#39;),
-                        attributes=manifest.get(&#39;attributes&#39;)
-                        )
+            table_def = cls.build_output_definition(
+                # helper parameters
+                stage=stage,
+                force_legacy_mode=force_legacy_mode,
+                is_sliced=is_sliced,
+                full_path=full_path,
+
+                # basic out manifest parameters
+                name=name,
+                destination=manifest.get(&#39;destination&#39;),
+                schema=cls.return_schema_from_manifest(manifest),
+                incremental=manifest.get(&#39;incremental&#39;),
+                primary_key=manifest.get(&#39;primary_key&#39;),
+                write_always=manifest.get(&#39;write_always&#39;, False),
+                delimiter=manifest.get(&#39;delimiter&#39;, &#39;,&#39;),
+                enclosure=manifest.get(&#39;enclosure&#39;, &#39;&#34;&#39;),
+                table_metadata=TableMetadata(manifest),
+
+                # additional in manifest parameters
+                delete_where_values=manifest.get(&#39;delete_where_values&#39;),
+                delete_where_column=manifest.get(&#39;delete_where_column&#39;),
+                delete_where_operator=manifest.get(&#39;delete_where_operator&#39;)
+            )
 
         return table_def
 
@@ -2838,7 +2885,7 @@ <h2 id="args">Args</h2>
 table with no input manifest.</p></div>
 </dd>
 <dt id="keboola.component.dao.TableDefinition.build_input_definition"><code class="name flex">
-<span>def <span class="ident">build_input_definition</span></span>(<span>name: str, full_path: Optional[Union[str, None]] = None, is_sliced: Optional[bool] = False, destination: Optional[str] = '', primary_key: Optional[List[str]] = None, columns: Optional[List[str]] = None, incremental: Optional[bool] = None, table_metadata: Optional[<a title="keboola.component.dao.TableMetadata" href="#keboola.component.dao.TableMetadata">TableMetadata</a>] = None, enclosure: Optional[str] = '"', delimiter: Optional[str] = ',', delete_where: Optional[dict] = None, stage: Optional[str] = 'in', write_always: Optional[bool] = False, schema: Optional[List[<a title="keboola.component.dao.ColumnDefinition" href="#keboola.component.dao.ColumnDefinition">ColumnDefinition</a>]] = None, rows_count: Optional[int] = None, data_size_bytes: Optional[int] = None, is_alias: Optional[bool] = False, uri: Optional[str] = None, id: Optional[str] = '', created: Optional[str] = None, last_change_date: Optional[str] = None, last_import_date: Optional[str] = None)</span>
+<span>def <span class="ident">build_input_definition</span></span>(<span>name: str, full_path: Optional[Union[str, None]] = None, is_sliced: Optional[bool] = False, destination: Optional[str] = '', primary_key: Optional[List[str]] = None, columns: Optional[List[str]] = None, incremental: Optional[bool] = None, table_metadata: Optional[<a title="keboola.component.dao.TableMetadata" href="#keboola.component.dao.TableMetadata">TableMetadata</a>] = None, enclosure: Optional[str] = '"', delimiter: Optional[str] = ',', delete_where: Optional[dict] = None, stage: Optional[str] = 'in', write_always: Optional[bool] = False, schema: Optional[Union[TypeOrderedDict[str, <a title="keboola.component.dao.ColumnDefinition" href="#keboola.component.dao.ColumnDefinition">ColumnDefinition</a>], list[str]]] = None, rows_count: Optional[int] = None, data_size_bytes: Optional[int] = None, is_alias: Optional[bool] = False, uri: Optional[str] = None, id: Optional[str] = '', created: Optional[str] = None, last_change_date: Optional[str] = None, last_import_date: Optional[str] = None, **kwargs)</span>
 </code></dt>
 <dd>
 <div class="desc"><p>Factory method for creating a TableDefinition instance for input tables.</p>
@@ -2900,7 +2947,7 @@ <h2 id="returns">Returns</h2>
 </dl></div>
 </dd>
 <dt id="keboola.component.dao.TableDefinition.build_output_definition"><code class="name flex">
-<span>def <span class="ident">build_output_definition</span></span>(<span>name: str, destination: Optional[str] = '', columns: Optional[List[str]] = None, primary_key: Optional[List[str]] = None, incremental: Optional[bool] = False, table_metadata: Optional[<a title="keboola.component.dao.TableMetadata" href="#keboola.component.dao.TableMetadata">TableMetadata</a>] = None, enclosure: Optional[str] = '"', delimiter: Optional[str] = ',', delete_where: Optional[dict] = None, write_always: Optional[bool] = False, schema: Optional[List[<a title="keboola.component.dao.ColumnDefinition" href="#keboola.component.dao.ColumnDefinition">ColumnDefinition</a>]] = None)</span>
+<span>def <span class="ident">build_output_definition</span></span>(<span>name: str, destination: Optional[str] = '', columns: Optional[List[str]] = None, primary_key: Optional[List[str]] = None, incremental: Optional[bool] = False, table_metadata: Optional[<a title="keboola.component.dao.TableMetadata" href="#keboola.component.dao.TableMetadata">TableMetadata</a>] = None, enclosure: Optional[str] = '"', delimiter: Optional[str] = ',', delete_where: Optional[dict] = None, write_always: Optional[bool] = False, schema: Optional[Union[TypeOrderedDict[str, <a title="keboola.component.dao.ColumnDefinition" href="#keboola.component.dao.ColumnDefinition">ColumnDefinition</a>], list[str]]] = None, **kwargs)</span>
 </code></dt>
 <dd>
 <div class="desc"><p>Factory method for creating a TableDefinition instance for output tables.</p>

setup.py

@@ -13,7 +13,7 @@
 
 setuptools.setup(
     name="keboola.component",
-    version="1.6.5",
+    version="1.6.6",
     author="Keboola KDS Team",
     project_urls=project_urls,
     setup_requires=['pytest-runner', 'flake8'],

src/keboola/component/dao.py

@@ -849,6 +849,15 @@ def __init__(self, name: str,
         self.delete_where_column = None
         self.delete_where_operator = None
 
+        if kwargs.get('delete_where_values'):
+            self.delete_where_values = kwargs['delete_where_values']
+
+        if kwargs.get('delete_where_column'):
+            self.delete_where_column = kwargs['delete_where_column']
+
+        if kwargs.get('delete_where_operator'):
+            self.delete_where_operator = kwargs['delete_where_operator']
+
         self.set_delete_where_from_dict(delete_where)
         self.write_always = write_always
 
@@ -883,7 +892,8 @@ def build_output_definition(cls, name: str,
                                 delimiter: Optional[str] = ',',
                                 delete_where: Optional[dict] = None,
                                 write_always: Optional[bool] = False,
-                                schema: Optional[List[ColumnDefinition]] = None,
+                                schema: Optional[Union[TypeOrderedDict[str, ColumnDefinition], list[str]]] = None,
+                                **kwargs
                                 ):
         """
         Factory method for creating a TableDefinition instance for output tables.
@@ -918,6 +928,7 @@ def build_output_definition(cls, name: str,
                    delete_where=delete_where,
                    write_always=write_always,
                    schema=schema,
+                   **kwargs
                    )
 
     @classmethod
@@ -935,7 +946,7 @@ def build_input_definition(cls, name: str,
                                delete_where: Optional[dict] = None,
                                stage: Optional[str] = 'in',
                                write_always: Optional[bool] = False,
-                               schema: Optional[List[ColumnDefinition]] = None,
+                               schema: Optional[Union[TypeOrderedDict[str, ColumnDefinition], list[str]]] = None,
                                rows_count: Optional[int] = None,
                                data_size_bytes: Optional[int] = None,
                                is_alias: Optional[bool] = False,
@@ -945,7 +956,9 @@ def build_input_definition(cls, name: str,
                                id: Optional[str] = '',
                                created: Optional[str] = None,
                                last_change_date: Optional[str] = None,
-                               last_import_date: Optional[str] = None):
+                               last_import_date: Optional[str] = None,
+                               **kwargs
+                               ):
         """
         Factory method for creating a TableDefinition instance for input tables.
 
@@ -1000,7 +1013,9 @@ def build_input_definition(cls, name: str,
                    id=id,
                    created=created,
                    last_change_date=last_change_date,
-                   last_import_date=last_import_date)
+                   last_import_date=last_import_date,
+                   **kwargs
+                   )
 
     @classmethod
     def convert_to_column_definition(cls, column_name, column_metadata, primary_key=False):
@@ -1088,6 +1103,9 @@ def build_from_manifest(cls,
         else:
             name = Path(manifest_file_path).stem
 
+        if manifest.get('name'):
+            name = manifest.get('name')
+
         # test if the manifest is output and incompatible
         force_legacy_mode = False
         if not manifest.get('columns') and manifest.get('primary_key'):
@@ -1097,28 +1115,57 @@ def build_from_manifest(cls,
 
         if manifest.get('id'):
             stage = 'in'
+            table_def = cls.build_input_definition(
+                # helper parameters
+                stage=stage,
+                force_legacy_mode=force_legacy_mode,
+                is_sliced=is_sliced,
+                full_path=full_path,
+
+                # basic in manifest parameters
+                id=manifest.get('id'),
+                uri=manifest.get('uri'),
+                name=name,
+                primary_key=manifest.get('primary_key'),
+                created=manifest.get('created'),
+                last_change_date=manifest.get('last_change_date'),
+                last_import_date=manifest.get('last_import_date'),
+                schema=cls.return_schema_from_manifest(manifest),
+                table_metadata=TableMetadata(manifest),
+
+                # additional in manifest parameters
+                rows_count=manifest.get('rows_count'),
+                data_size_bytes=manifest.get('data_size_bytes'),
+                is_alias=manifest.get('is_alias'),
+                attributes=manifest.get('attributes'),
+                indexed_columns=manifest.get('indexed_columns'),
+            )
+
         else:
             stage = 'out'
-
-        table_def = cls(name=name,
-                        stage=stage,
-                        full_path=full_path,
-                        is_sliced=is_sliced,
-                        id=manifest.get('id'),
-                        table_metadata=TableMetadata(manifest),
-                        primary_key=manifest.get('primary_key'),
-                        schema=cls.return_schema_from_manifest(manifest),
-                        uri=manifest.get('uri'),
-                        created=manifest.get('created'),
-                        last_change_date=manifest.get('last_change_date'),
-                        last_import_date=manifest.get('last_import_date'),
-                        rows_count=manifest.get('rows_count'),
-                        data_size_bytes=manifest.get('data_size_bytes'),
-                        is_alias=manifest.get('is_alias'),
-                        force_legacy_mode=force_legacy_mode,
-                        indexed_columns=manifest.get('indexed_columns'),
-                        attributes=manifest.get('attributes')
-                        )
+            table_def = cls.build_output_definition(
+                # helper parameters
+                stage=stage,
+                force_legacy_mode=force_legacy_mode,
+                is_sliced=is_sliced,
+                full_path=full_path,
+
+                # basic out manifest parameters
+                name=name,
+                destination=manifest.get('destination'),
+                schema=cls.return_schema_from_manifest(manifest),
+                incremental=manifest.get('incremental'),
+                primary_key=manifest.get('primary_key'),
+                write_always=manifest.get('write_always', False),
+                delimiter=manifest.get('delimiter', ','),
+                enclosure=manifest.get('enclosure', '"'),
+                table_metadata=TableMetadata(manifest),
+
+                # additional in manifest parameters
+                delete_where_values=manifest.get('delete_where_values'),
+                delete_where_column=manifest.get('delete_where_column'),
+                delete_where_operator=manifest.get('delete_where_operator')
+            )
 
         return table_def
 

tests/data_examples/data1/in/tables/fooBar.manifest

@@ -1,7 +1,7 @@
 {
     "id": "in.c-main.test2",
     "uri": "https:\/\/connection.keboola.com\/\/v2\/storage\/tables\/in.c-main.test2",
-    "name": "test2",
+    "name": "fooBar",
     "primary_key": [],
     "indexed_columns": [],
     "created": "2015-09-22T17:04:52+0200",

tests/data_examples/data1/in/tables/orphaned_manifest.csv.manifest

@@ -1,7 +1,7 @@
 {
     "id": "in.c-main.test",
     "uri": "https:\/\/connection.keboola.com\/\/v2\/storage\/tables\/in.c-main.test",
-    "name": "test",
+    "name": "orphaned_manifest.csv",
     "primary_key": [],
     "indexed_columns": [],
     "created": "2015-11-02T09:11:37+0100",

tests/data_examples/data1/in/tables/sample.csv.manifest

@@ -1,7 +1,7 @@
 {
     "id": "in.c-main.test",
     "uri": "https:\/\/connection.keboola.com\/\/v2\/storage\/tables\/in.c-main.test",
-    "name": "test",
+    "name": "sample.csv",
     "primary_key": [],
     "indexed_columns": [],
     "created": "2015-11-02T09:11:37+0100",