From 9c1c6b7365bdac8d527be1b70bfff0af6e96b45e Mon Sep 17 00:00:00 2001 From: h3n4l Date: Wed, 15 Apr 2026 16:36:16 +0800 Subject: [PATCH] =?UTF-8?q?feat(doris):=20T5.3=20storage=20infrastructure?= =?UTF-8?q?=20DDL=20=E2=80=94=20vault,=20policy,=20repository,=20stage,=20?= =?UTF-8?q?file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add AST nodes, parser, and full dispatch for all Doris storage-related DDL: STORAGE VAULT (CREATE/ALTER/DROP + SET/UNSET DEFAULT), STORAGE POLICY (CREATE/ALTER/DROP), REPOSITORY (CREATE with S3/HDFS/BROKER/READ ONLY, ALTER, DROP), STAGE (CREATE/DROP), and FILE (CREATE IN db, DROP FROM db). 47 tests pass covering legacy corpus round-trips and all syntax forms. Co-Authored-By: Claude Sonnet 4.6 --- doris/ast/ddlnodes.go | 237 ++++++++++ doris/ast/loc.go | 30 ++ doris/ast/nodetags.go | 85 ++++ doris/ast/walk_children.go | 58 +++ doris/parser/parser.go | 42 +- doris/parser/storage.go | 601 ++++++++++++++++++++++++ doris/parser/storage_test.go | 882 +++++++++++++++++++++++++++++++++++ 7 files changed, 1934 insertions(+), 1 deletion(-) create mode 100644 doris/parser/storage.go create mode 100644 doris/parser/storage_test.go diff --git a/doris/ast/ddlnodes.go b/doris/ast/ddlnodes.go index 598e34d4..2cea88d1 100644 --- a/doris/ast/ddlnodes.go +++ b/doris/ast/ddlnodes.go @@ -368,3 +368,240 @@ type DropViewStmt struct { func (n *DropViewStmt) Tag() NodeTag { return T_DropViewStmt } var _ Node = (*DropViewStmt)(nil) + +// --------------------------------------------------------------------------- +// STORAGE VAULT DDL nodes (T5.3) +// --------------------------------------------------------------------------- + +// CreateStorageVaultStmt represents: +// +// CREATE STORAGE VAULT [IF NOT EXISTS] name PROPERTIES(...) +type CreateStorageVaultStmt struct { + Name string + IfNotExists bool + Properties []*Property + Loc Loc +} + +// Tag implements Node. +func (n *CreateStorageVaultStmt) Tag() NodeTag { return T_CreateStorageVaultStmt } + +var _ Node = (*CreateStorageVaultStmt)(nil) + +// AlterStorageVaultStmt represents: +// +// ALTER STORAGE VAULT name PROPERTIES(...) +type AlterStorageVaultStmt struct { + Name string + Properties []*Property + Loc Loc +} + +// Tag implements Node. +func (n *AlterStorageVaultStmt) Tag() NodeTag { return T_AlterStorageVaultStmt } + +var _ Node = (*AlterStorageVaultStmt)(nil) + +// DropStorageVaultStmt represents: +// +// DROP STORAGE VAULT [IF EXISTS] name +type DropStorageVaultStmt struct { + Name string + IfExists bool + Loc Loc +} + +// Tag implements Node. +func (n *DropStorageVaultStmt) Tag() NodeTag { return T_DropStorageVaultStmt } + +var _ Node = (*DropStorageVaultStmt)(nil) + +// SetDefaultStorageVaultStmt represents: +// +// SET DEFAULT STORAGE VAULT name +type SetDefaultStorageVaultStmt struct { + Name string + Loc Loc +} + +// Tag implements Node. +func (n *SetDefaultStorageVaultStmt) Tag() NodeTag { return T_SetDefaultStorageVaultStmt } + +var _ Node = (*SetDefaultStorageVaultStmt)(nil) + +// UnsetDefaultStorageVaultStmt represents: +// +// UNSET DEFAULT STORAGE VAULT +type UnsetDefaultStorageVaultStmt struct { + Loc Loc +} + +// Tag implements Node. +func (n *UnsetDefaultStorageVaultStmt) Tag() NodeTag { return T_UnsetDefaultStorageVaultStmt } + +var _ Node = (*UnsetDefaultStorageVaultStmt)(nil) + +// --------------------------------------------------------------------------- +// STORAGE POLICY DDL nodes (T5.3) +// --------------------------------------------------------------------------- + +// CreateStoragePolicyStmt represents: +// +// CREATE STORAGE POLICY [IF NOT EXISTS] name PROPERTIES(...) +type CreateStoragePolicyStmt struct { + Name string + IfNotExists bool + Properties []*Property + Loc Loc +} + +// Tag implements Node. +func (n *CreateStoragePolicyStmt) Tag() NodeTag { return T_CreateStoragePolicyStmt } + +var _ Node = (*CreateStoragePolicyStmt)(nil) + +// AlterStoragePolicyStmt represents: +// +// ALTER STORAGE POLICY name PROPERTIES(...) +type AlterStoragePolicyStmt struct { + Name string + Properties []*Property + Loc Loc +} + +// Tag implements Node. +func (n *AlterStoragePolicyStmt) Tag() NodeTag { return T_AlterStoragePolicyStmt } + +var _ Node = (*AlterStoragePolicyStmt)(nil) + +// DropStoragePolicyStmt represents: +// +// DROP STORAGE POLICY [IF EXISTS] name +type DropStoragePolicyStmt struct { + Name string + IfExists bool + Loc Loc +} + +// Tag implements Node. +func (n *DropStoragePolicyStmt) Tag() NodeTag { return T_DropStoragePolicyStmt } + +var _ Node = (*DropStoragePolicyStmt)(nil) + +// --------------------------------------------------------------------------- +// REPOSITORY DDL nodes (T5.3) +// --------------------------------------------------------------------------- + +// CreateRepositoryStmt represents: +// +// CREATE [READ ONLY] REPOSITORY name +// WITH {S3 | HDFS | BROKER broker_name} +// ON LOCATION "uri" +// PROPERTIES(...) +type CreateRepositoryStmt struct { + Name string + ReadOnly bool + Type string // "S3", "HDFS", or "BROKER" + BrokerName string // for BROKER form only + Properties []*Property + Loc Loc +} + +// Tag implements Node. +func (n *CreateRepositoryStmt) Tag() NodeTag { return T_CreateRepositoryStmt } + +var _ Node = (*CreateRepositoryStmt)(nil) + +// AlterRepositoryStmt represents: +// +// ALTER REPOSITORY name PROPERTIES(...) +type AlterRepositoryStmt struct { + Name string + Properties []*Property + Loc Loc +} + +// Tag implements Node. +func (n *AlterRepositoryStmt) Tag() NodeTag { return T_AlterRepositoryStmt } + +var _ Node = (*AlterRepositoryStmt)(nil) + +// DropRepositoryStmt represents: +// +// DROP REPOSITORY name +type DropRepositoryStmt struct { + Name string + Loc Loc +} + +// Tag implements Node. +func (n *DropRepositoryStmt) Tag() NodeTag { return T_DropRepositoryStmt } + +var _ Node = (*DropRepositoryStmt)(nil) + +// --------------------------------------------------------------------------- +// STAGE DDL nodes (T5.3) +// --------------------------------------------------------------------------- + +// CreateStageStmt represents: +// +// CREATE STAGE [IF NOT EXISTS] name PROPERTIES(...) +type CreateStageStmt struct { + Name string + IfNotExists bool + Properties []*Property + Loc Loc +} + +// Tag implements Node. +func (n *CreateStageStmt) Tag() NodeTag { return T_CreateStageStmt } + +var _ Node = (*CreateStageStmt)(nil) + +// DropStageStmt represents: +// +// DROP STAGE [IF EXISTS] name +type DropStageStmt struct { + Name string + IfExists bool + Loc Loc +} + +// Tag implements Node. +func (n *DropStageStmt) Tag() NodeTag { return T_DropStageStmt } + +var _ Node = (*DropStageStmt)(nil) + +// --------------------------------------------------------------------------- +// FILE DDL nodes (T5.3) +// --------------------------------------------------------------------------- + +// CreateFileStmt represents: +// +// CREATE FILE file_name [IN db] PROPERTIES(...) +type CreateFileStmt struct { + Name string + Database string // optional IN db_name + Properties []*Property + Loc Loc +} + +// Tag implements Node. +func (n *CreateFileStmt) Tag() NodeTag { return T_CreateFileStmt } + +var _ Node = (*CreateFileStmt)(nil) + +// DropFileStmt represents: +// +// DROP FILE file_name [FROM db] PROPERTIES(...) +type DropFileStmt struct { + Name string + Database string // optional FROM db_name + Properties []*Property + Loc Loc +} + +// Tag implements Node. +func (n *DropFileStmt) Tag() NodeTag { return T_DropFileStmt } + +var _ Node = (*DropFileStmt)(nil) diff --git a/doris/ast/loc.go b/doris/ast/loc.go index cfc31c2d..69ad5640 100644 --- a/doris/ast/loc.go +++ b/doris/ast/loc.go @@ -124,6 +124,36 @@ func NodeLoc(n Node) Loc { return v.Loc case *MergeClause: return v.Loc + case *CreateStorageVaultStmt: + return v.Loc + case *AlterStorageVaultStmt: + return v.Loc + case *DropStorageVaultStmt: + return v.Loc + case *SetDefaultStorageVaultStmt: + return v.Loc + case *UnsetDefaultStorageVaultStmt: + return v.Loc + case *CreateStoragePolicyStmt: + return v.Loc + case *AlterStoragePolicyStmt: + return v.Loc + case *DropStoragePolicyStmt: + return v.Loc + case *CreateRepositoryStmt: + return v.Loc + case *AlterRepositoryStmt: + return v.Loc + case *DropRepositoryStmt: + return v.Loc + case *CreateStageStmt: + return v.Loc + case *DropStageStmt: + return v.Loc + case *CreateFileStmt: + return v.Loc + case *DropFileStmt: + return v.Loc default: return NoLoc() } diff --git a/doris/ast/nodetags.go b/doris/ast/nodetags.go index 1cc7c3e6..f95e7334 100644 --- a/doris/ast/nodetags.go +++ b/doris/ast/nodetags.go @@ -206,6 +206,61 @@ const ( // T_MergeClause is the tag for *MergeClause (one WHEN clause inside MERGE). T_MergeClause + + // DDL — STORAGE VAULT nodes (T5.3). + + // T_CreateStorageVaultStmt is the tag for *CreateStorageVaultStmt. + T_CreateStorageVaultStmt + + // T_AlterStorageVaultStmt is the tag for *AlterStorageVaultStmt. + T_AlterStorageVaultStmt + + // T_DropStorageVaultStmt is the tag for *DropStorageVaultStmt. + T_DropStorageVaultStmt + + // T_SetDefaultStorageVaultStmt is the tag for *SetDefaultStorageVaultStmt. + T_SetDefaultStorageVaultStmt + + // T_UnsetDefaultStorageVaultStmt is the tag for *UnsetDefaultStorageVaultStmt. + T_UnsetDefaultStorageVaultStmt + + // DDL — STORAGE POLICY nodes (T5.3). + + // T_CreateStoragePolicyStmt is the tag for *CreateStoragePolicyStmt. + T_CreateStoragePolicyStmt + + // T_AlterStoragePolicyStmt is the tag for *AlterStoragePolicyStmt. + T_AlterStoragePolicyStmt + + // T_DropStoragePolicyStmt is the tag for *DropStoragePolicyStmt. + T_DropStoragePolicyStmt + + // DDL — REPOSITORY nodes (T5.3). + + // T_CreateRepositoryStmt is the tag for *CreateRepositoryStmt. + T_CreateRepositoryStmt + + // T_AlterRepositoryStmt is the tag for *AlterRepositoryStmt. + T_AlterRepositoryStmt + + // T_DropRepositoryStmt is the tag for *DropRepositoryStmt. + T_DropRepositoryStmt + + // DDL — STAGE nodes (T5.3). + + // T_CreateStageStmt is the tag for *CreateStageStmt. + T_CreateStageStmt + + // T_DropStageStmt is the tag for *DropStageStmt. + T_DropStageStmt + + // DDL — FILE nodes (T5.3). + + // T_CreateFileStmt is the tag for *CreateFileStmt. + T_CreateFileStmt + + // T_DropFileStmt is the tag for *DropFileStmt. + T_DropFileStmt ) // String returns a human-readable representation of the tag. @@ -327,6 +382,36 @@ func (t NodeTag) String() string { return "MergeStmt" case T_MergeClause: return "MergeClause" + case T_CreateStorageVaultStmt: + return "CreateStorageVaultStmt" + case T_AlterStorageVaultStmt: + return "AlterStorageVaultStmt" + case T_DropStorageVaultStmt: + return "DropStorageVaultStmt" + case T_SetDefaultStorageVaultStmt: + return "SetDefaultStorageVaultStmt" + case T_UnsetDefaultStorageVaultStmt: + return "UnsetDefaultStorageVaultStmt" + case T_CreateStoragePolicyStmt: + return "CreateStoragePolicyStmt" + case T_AlterStoragePolicyStmt: + return "AlterStoragePolicyStmt" + case T_DropStoragePolicyStmt: + return "DropStoragePolicyStmt" + case T_CreateRepositoryStmt: + return "CreateRepositoryStmt" + case T_AlterRepositoryStmt: + return "AlterRepositoryStmt" + case T_DropRepositoryStmt: + return "DropRepositoryStmt" + case T_CreateStageStmt: + return "CreateStageStmt" + case T_DropStageStmt: + return "DropStageStmt" + case T_CreateFileStmt: + return "CreateFileStmt" + case T_DropFileStmt: + return "DropFileStmt" default: return "Unknown" } diff --git a/doris/ast/walk_children.go b/doris/ast/walk_children.go index e3b3c3c7..22f77225 100644 --- a/doris/ast/walk_children.go +++ b/doris/ast/walk_children.go @@ -348,5 +348,63 @@ func walkChildren(v Visitor, node Node) { Walk(v, val) } } + + // DDL — STORAGE VAULT nodes (T5.3). + case *CreateStorageVaultStmt: + for _, prop := range n.Properties { + Walk(v, prop) + } + case *AlterStorageVaultStmt: + for _, prop := range n.Properties { + Walk(v, prop) + } + case *DropStorageVaultStmt: + // leaf-ish node, no Node children + case *SetDefaultStorageVaultStmt: + // leaf-ish node, no Node children + case *UnsetDefaultStorageVaultStmt: + // leaf-ish node, no Node children + + // DDL — STORAGE POLICY nodes (T5.3). + case *CreateStoragePolicyStmt: + for _, prop := range n.Properties { + Walk(v, prop) + } + case *AlterStoragePolicyStmt: + for _, prop := range n.Properties { + Walk(v, prop) + } + case *DropStoragePolicyStmt: + // leaf-ish node, no Node children + + // DDL — REPOSITORY nodes (T5.3). + case *CreateRepositoryStmt: + for _, prop := range n.Properties { + Walk(v, prop) + } + case *AlterRepositoryStmt: + for _, prop := range n.Properties { + Walk(v, prop) + } + case *DropRepositoryStmt: + // leaf-ish node, no Node children + + // DDL — STAGE nodes (T5.3). + case *CreateStageStmt: + for _, prop := range n.Properties { + Walk(v, prop) + } + case *DropStageStmt: + // leaf-ish node, no Node children + + // DDL — FILE nodes (T5.3). + case *CreateFileStmt: + for _, prop := range n.Properties { + Walk(v, prop) + } + case *DropFileStmt: + for _, prop := range n.Properties { + Walk(v, prop) + } } } diff --git a/doris/parser/parser.go b/doris/parser/parser.go index 1d23e24b..11ac3292 100644 --- a/doris/parser/parser.go +++ b/doris/parser/parser.go @@ -190,11 +190,27 @@ func (p *Parser) parseStmt() (ast.Node, error) { return nil, err } return p.parseCreateView(createTok.Loc, true) + case kwSTORAGE: + p.advance() // consume STORAGE + return p.parseCreateStorage(createTok.Loc) + case kwREPOSITORY: + return p.parseCreateRepository(createTok.Loc, false) + case kwREAD: + // CREATE READ ONLY REPOSITORY ... + p.advance() // consume READ + if _, err := p.expect(kwONLY); err != nil { + return nil, err + } + return p.parseCreateRepository(createTok.Loc, true) + case kwSTAGE: + return p.parseCreateStage(createTok.Loc) + case kwFILE: + return p.parseCreateFile(createTok.Loc) default: return p.unsupported("CREATE") } case kwALTER: - p.advance() // consume ALTER; cur is now the object type keyword + alterTok := p.advance() // consume ALTER; cur is now the object type keyword switch p.cur.Kind { case kwDATABASE, kwSCHEMA: return p.parseAlterDatabase() @@ -202,6 +218,11 @@ func (p *Parser) parseStmt() (ast.Node, error) { return p.parseAlterTable() case kwVIEW: return p.parseAlterView() + case kwSTORAGE: + p.advance() // consume STORAGE + return p.parseAlterStorage(alterTok.Loc) + case kwREPOSITORY: + return p.parseAlterRepository(alterTok.Loc) default: return p.unsupported("ALTER") } @@ -214,6 +235,15 @@ func (p *Parser) parseStmt() (ast.Node, error) { return p.parseDropDatabase() case kwVIEW: return p.parseDropView(dropTok.Loc) + case kwSTORAGE: + p.advance() // consume STORAGE + return p.parseDropStorage(dropTok.Loc) + case kwREPOSITORY: + return p.parseDropRepository(dropTok.Loc) + case kwSTAGE: + return p.parseDropStage(dropTok.Loc) + case kwFILE: + return p.parseDropFile(dropTok.Loc) default: return p.unsupported("DROP") } @@ -279,8 +309,18 @@ func (p *Parser) parseStmt() (ast.Node, error) { // Set / Unset case kwSET: + setTok := p.advance() // consume SET + // SET DEFAULT STORAGE VAULT name + if p.cur.Kind == kwDEFAULT && p.peekNext().Kind == kwSTORAGE { + return p.parseSetDefaultStorageVault(setTok.Loc) + } return p.unsupported("SET") case kwUNSET: + unsetTok := p.advance() // consume UNSET + // UNSET DEFAULT STORAGE VAULT + if p.cur.Kind == kwDEFAULT && p.peekNext().Kind == kwSTORAGE { + return p.parseUnsetDefaultStorageVault(unsetTok.Loc) + } return p.unsupported("UNSET") // Admin / System diff --git a/doris/parser/storage.go b/doris/parser/storage.go new file mode 100644 index 00000000..507b10be --- /dev/null +++ b/doris/parser/storage.go @@ -0,0 +1,601 @@ +package parser + +import ( + "github.com/bytebase/omni/doris/ast" +) + +// --------------------------------------------------------------------------- +// STORAGE VAULT +// --------------------------------------------------------------------------- + +// parseCreateStorageVault parses: +// +// CREATE STORAGE VAULT [IF NOT EXISTS] name PROPERTIES(...) +// +// CREATE and STORAGE have already been consumed; cur is VAULT. +func (p *Parser) parseCreateStorageVault(startLoc ast.Loc) (ast.Node, error) { + p.advance() // consume VAULT + + stmt := &ast.CreateStorageVaultStmt{} + + // Optional IF NOT EXISTS + if p.cur.Kind == kwIF { + p.advance() // consume IF + if _, err := p.expect(kwNOT); err != nil { + return nil, err + } + if _, err := p.expect(kwEXISTS); err != nil { + return nil, err + } + stmt.IfNotExists = true + } + + name, nameLoc, err := p.parseIdentifier() + if err != nil { + return nil, err + } + stmt.Name = name + endLoc := nameLoc + + if p.cur.Kind == kwPROPERTIES { + props, err := p.parseProperties() + if err != nil { + return nil, err + } + stmt.Properties = props + if len(props) > 0 { + endLoc = ast.NodeLoc(props[len(props)-1]) + } + } + + stmt.Loc = startLoc.Merge(endLoc) + return stmt, nil +} + +// parseAlterStorageVault parses: +// +// ALTER STORAGE VAULT name PROPERTIES(...) +// +// ALTER and STORAGE have already been consumed; cur is VAULT. +func (p *Parser) parseAlterStorageVault(startLoc ast.Loc) (ast.Node, error) { + p.advance() // consume VAULT + + name, nameLoc, err := p.parseIdentifier() + if err != nil { + return nil, err + } + endLoc := nameLoc + + stmt := &ast.AlterStorageVaultStmt{Name: name} + + if p.cur.Kind == kwPROPERTIES { + props, err := p.parseProperties() + if err != nil { + return nil, err + } + stmt.Properties = props + if len(props) > 0 { + endLoc = ast.NodeLoc(props[len(props)-1]) + } + } + + stmt.Loc = startLoc.Merge(endLoc) + return stmt, nil +} + +// parseDropStorageVault parses: +// +// DROP STORAGE VAULT [IF EXISTS] name +// +// DROP and STORAGE have already been consumed; cur is VAULT. +func (p *Parser) parseDropStorageVault(startLoc ast.Loc) (ast.Node, error) { + p.advance() // consume VAULT + + stmt := &ast.DropStorageVaultStmt{} + + if p.cur.Kind == kwIF { + p.advance() // consume IF + if _, err := p.expect(kwEXISTS); err != nil { + return nil, err + } + stmt.IfExists = true + } + + name, nameLoc, err := p.parseIdentifier() + if err != nil { + return nil, err + } + stmt.Name = name + stmt.Loc = startLoc.Merge(nameLoc) + return stmt, nil +} + +// parseSetDefaultStorageVault parses: +// +// SET DEFAULT STORAGE VAULT name +// +// SET has already been consumed; cur is DEFAULT. +func (p *Parser) parseSetDefaultStorageVault(startLoc ast.Loc) (ast.Node, error) { + p.advance() // consume DEFAULT + if _, err := p.expect(kwSTORAGE); err != nil { + return nil, err + } + if _, err := p.expect(kwVAULT); err != nil { + return nil, err + } + + name, nameLoc, err := p.parseIdentifier() + if err != nil { + return nil, err + } + + stmt := &ast.SetDefaultStorageVaultStmt{ + Name: name, + Loc: startLoc.Merge(nameLoc), + } + return stmt, nil +} + +// parseUnsetDefaultStorageVault parses: +// +// UNSET DEFAULT STORAGE VAULT +// +// UNSET has already been consumed; cur is DEFAULT. +func (p *Parser) parseUnsetDefaultStorageVault(startLoc ast.Loc) (ast.Node, error) { + p.advance() // consume DEFAULT + if _, err := p.expect(kwSTORAGE); err != nil { + return nil, err + } + vaultTok, err := p.expect(kwVAULT) + if err != nil { + return nil, err + } + + stmt := &ast.UnsetDefaultStorageVaultStmt{ + Loc: startLoc.Merge(vaultTok.Loc), + } + return stmt, nil +} + +// --------------------------------------------------------------------------- +// STORAGE POLICY +// --------------------------------------------------------------------------- + +// parseCreateStoragePolicy parses: +// +// CREATE STORAGE POLICY [IF NOT EXISTS] name PROPERTIES(...) +// +// CREATE and STORAGE have already been consumed; cur is POLICY. +func (p *Parser) parseCreateStoragePolicy(startLoc ast.Loc) (ast.Node, error) { + p.advance() // consume POLICY + + stmt := &ast.CreateStoragePolicyStmt{} + + if p.cur.Kind == kwIF { + p.advance() // consume IF + if _, err := p.expect(kwNOT); err != nil { + return nil, err + } + if _, err := p.expect(kwEXISTS); err != nil { + return nil, err + } + stmt.IfNotExists = true + } + + name, nameLoc, err := p.parseIdentifier() + if err != nil { + return nil, err + } + stmt.Name = name + endLoc := nameLoc + + if p.cur.Kind == kwPROPERTIES { + props, err := p.parseProperties() + if err != nil { + return nil, err + } + stmt.Properties = props + if len(props) > 0 { + endLoc = ast.NodeLoc(props[len(props)-1]) + } + } + + stmt.Loc = startLoc.Merge(endLoc) + return stmt, nil +} + +// parseAlterStoragePolicy parses: +// +// ALTER STORAGE POLICY name PROPERTIES(...) +// +// ALTER and STORAGE have already been consumed; cur is POLICY. +func (p *Parser) parseAlterStoragePolicy(startLoc ast.Loc) (ast.Node, error) { + p.advance() // consume POLICY + + name, nameLoc, err := p.parseIdentifier() + if err != nil { + return nil, err + } + endLoc := nameLoc + + stmt := &ast.AlterStoragePolicyStmt{Name: name} + + if p.cur.Kind == kwPROPERTIES { + props, err := p.parseProperties() + if err != nil { + return nil, err + } + stmt.Properties = props + if len(props) > 0 { + endLoc = ast.NodeLoc(props[len(props)-1]) + } + } + + stmt.Loc = startLoc.Merge(endLoc) + return stmt, nil +} + +// parseDropStoragePolicy parses: +// +// DROP STORAGE POLICY [IF EXISTS] name +// +// DROP and STORAGE have already been consumed; cur is POLICY. +func (p *Parser) parseDropStoragePolicy(startLoc ast.Loc) (ast.Node, error) { + p.advance() // consume POLICY + + stmt := &ast.DropStoragePolicyStmt{} + + if p.cur.Kind == kwIF { + p.advance() // consume IF + if _, err := p.expect(kwEXISTS); err != nil { + return nil, err + } + stmt.IfExists = true + } + + name, nameLoc, err := p.parseIdentifier() + if err != nil { + return nil, err + } + stmt.Name = name + stmt.Loc = startLoc.Merge(nameLoc) + return stmt, nil +} + +// --------------------------------------------------------------------------- +// REPOSITORY +// --------------------------------------------------------------------------- + +// parseCreateRepository parses: +// +// CREATE [READ ONLY] REPOSITORY name +// WITH {S3 | HDFS | BROKER broker_name} +// ON LOCATION "uri" +// PROPERTIES(...) +// +// CREATE has already been consumed; cur is REPOSITORY (or READ). +// readOnly indicates whether READ ONLY was already consumed. +func (p *Parser) parseCreateRepository(startLoc ast.Loc, readOnly bool) (ast.Node, error) { + p.advance() // consume REPOSITORY + + name, _, err := p.parseIdentifier() + if err != nil { + return nil, err + } + + stmt := &ast.CreateRepositoryStmt{Name: name, ReadOnly: readOnly} + + // WITH S3 | HDFS | BROKER broker_name + if _, err := p.expect(kwWITH); err != nil { + return nil, err + } + + switch p.cur.Kind { + case kwS3: + stmt.Type = "S3" + p.advance() + case kwHDFS: + stmt.Type = "HDFS" + p.advance() + case kwBROKER: + stmt.Type = "BROKER" + p.advance() + // broker_name is an identifier following BROKER + brokerName, _, err := p.parseIdentifier() + if err != nil { + return nil, err + } + stmt.BrokerName = brokerName + default: + return nil, p.syntaxErrorAtCur() + } + + // ON LOCATION "uri" + if _, err := p.expect(kwON); err != nil { + return nil, err + } + if _, err := p.expect(kwLOCATION); err != nil { + return nil, err + } + // location is a string literal; consume it (we don't store it separately + // since it will appear in PROPERTIES as "location" key in practice, but the + // syntax places it here explicitly) + if p.cur.Kind != tokString { + return nil, p.syntaxErrorAtCur() + } + p.advance() // consume location URI string + + endLoc := p.prev.Loc + + // PROPERTIES(...) + if p.cur.Kind == kwPROPERTIES { + props, err := p.parseProperties() + if err != nil { + return nil, err + } + stmt.Properties = props + if len(props) > 0 { + endLoc = ast.NodeLoc(props[len(props)-1]) + } + } + + stmt.Loc = startLoc.Merge(endLoc) + return stmt, nil +} + +// parseAlterRepository parses: +// +// ALTER REPOSITORY name PROPERTIES(...) +// +// ALTER has already been consumed; cur is REPOSITORY. +func (p *Parser) parseAlterRepository(startLoc ast.Loc) (ast.Node, error) { + p.advance() // consume REPOSITORY + + name, nameLoc, err := p.parseIdentifier() + if err != nil { + return nil, err + } + endLoc := nameLoc + + stmt := &ast.AlterRepositoryStmt{Name: name} + + if p.cur.Kind == kwPROPERTIES { + props, err := p.parseProperties() + if err != nil { + return nil, err + } + stmt.Properties = props + if len(props) > 0 { + endLoc = ast.NodeLoc(props[len(props)-1]) + } + } + + stmt.Loc = startLoc.Merge(endLoc) + return stmt, nil +} + +// parseDropRepository parses: +// +// DROP REPOSITORY name +// +// DROP has already been consumed; cur is REPOSITORY. +func (p *Parser) parseDropRepository(startLoc ast.Loc) (ast.Node, error) { + p.advance() // consume REPOSITORY + + name, nameLoc, err := p.parseIdentifier() + if err != nil { + return nil, err + } + + stmt := &ast.DropRepositoryStmt{ + Name: name, + Loc: startLoc.Merge(nameLoc), + } + return stmt, nil +} + +// --------------------------------------------------------------------------- +// STAGE +// --------------------------------------------------------------------------- + +// parseCreateStage parses: +// +// CREATE STAGE [IF NOT EXISTS] name PROPERTIES(...) +// +// CREATE has already been consumed; cur is STAGE. +func (p *Parser) parseCreateStage(startLoc ast.Loc) (ast.Node, error) { + p.advance() // consume STAGE + + stmt := &ast.CreateStageStmt{} + + if p.cur.Kind == kwIF { + p.advance() // consume IF + if _, err := p.expect(kwNOT); err != nil { + return nil, err + } + if _, err := p.expect(kwEXISTS); err != nil { + return nil, err + } + stmt.IfNotExists = true + } + + name, nameLoc, err := p.parseIdentifier() + if err != nil { + return nil, err + } + stmt.Name = name + endLoc := nameLoc + + if p.cur.Kind == kwPROPERTIES { + props, err := p.parseProperties() + if err != nil { + return nil, err + } + stmt.Properties = props + if len(props) > 0 { + endLoc = ast.NodeLoc(props[len(props)-1]) + } + } + + stmt.Loc = startLoc.Merge(endLoc) + return stmt, nil +} + +// parseDropStage parses: +// +// DROP STAGE [IF EXISTS] name +// +// DROP has already been consumed; cur is STAGE. +func (p *Parser) parseDropStage(startLoc ast.Loc) (ast.Node, error) { + p.advance() // consume STAGE + + stmt := &ast.DropStageStmt{} + + if p.cur.Kind == kwIF { + p.advance() // consume IF + if _, err := p.expect(kwEXISTS); err != nil { + return nil, err + } + stmt.IfExists = true + } + + name, nameLoc, err := p.parseIdentifier() + if err != nil { + return nil, err + } + stmt.Name = name + stmt.Loc = startLoc.Merge(nameLoc) + return stmt, nil +} + +// --------------------------------------------------------------------------- +// FILE +// --------------------------------------------------------------------------- + +// parseCreateFile parses: +// +// CREATE FILE file_name [IN db] PROPERTIES(...) +// +// CREATE has already been consumed; cur is FILE. +func (p *Parser) parseCreateFile(startLoc ast.Loc) (ast.Node, error) { + p.advance() // consume FILE + + // file_name is a string literal or identifier + name, nameLoc, err := p.parseIdentifierOrString() + if err != nil { + return nil, err + } + endLoc := nameLoc + + stmt := &ast.CreateFileStmt{Name: name} + + // Optional IN db_name + if p.cur.Kind == kwIN { + p.advance() // consume IN + dbName, dbLoc, err := p.parseIdentifier() + if err != nil { + return nil, err + } + stmt.Database = dbName + endLoc = dbLoc + } + + if p.cur.Kind == kwPROPERTIES { + props, err := p.parseProperties() + if err != nil { + return nil, err + } + stmt.Properties = props + if len(props) > 0 { + endLoc = ast.NodeLoc(props[len(props)-1]) + } + } + + stmt.Loc = startLoc.Merge(endLoc) + return stmt, nil +} + +// parseDropFile parses: +// +// DROP FILE file_name [FROM db] PROPERTIES(...) +// +// DROP has already been consumed; cur is FILE. +func (p *Parser) parseDropFile(startLoc ast.Loc) (ast.Node, error) { + p.advance() // consume FILE + + name, nameLoc, err := p.parseIdentifierOrString() + if err != nil { + return nil, err + } + endLoc := nameLoc + + stmt := &ast.DropFileStmt{Name: name} + + // Optional FROM db_name + if p.cur.Kind == kwFROM { + p.advance() // consume FROM + dbName, dbLoc, err := p.parseIdentifier() + if err != nil { + return nil, err + } + stmt.Database = dbName + endLoc = dbLoc + } + + if p.cur.Kind == kwPROPERTIES { + props, err := p.parseProperties() + if err != nil { + return nil, err + } + stmt.Properties = props + if len(props) > 0 { + endLoc = ast.NodeLoc(props[len(props)-1]) + } + } + + stmt.Loc = startLoc.Merge(endLoc) + return stmt, nil +} + +// --------------------------------------------------------------------------- +// Shared dispatch helpers called from parser.go +// --------------------------------------------------------------------------- + +// parseCreateStorage dispatches on the VAULT|POLICY token after STORAGE. +// CREATE and STORAGE have already been consumed; cur is VAULT or POLICY. +func (p *Parser) parseCreateStorage(startLoc ast.Loc) (ast.Node, error) { + switch p.cur.Kind { + case kwVAULT: + return p.parseCreateStorageVault(startLoc) + case kwPOLICY: + return p.parseCreateStoragePolicy(startLoc) + default: + return p.unsupported("CREATE STORAGE") + } +} + +// parseAlterStorage dispatches on VAULT|POLICY after STORAGE. +// ALTER and STORAGE have already been consumed; cur is VAULT or POLICY. +func (p *Parser) parseAlterStorage(startLoc ast.Loc) (ast.Node, error) { + switch p.cur.Kind { + case kwVAULT: + return p.parseAlterStorageVault(startLoc) + case kwPOLICY: + return p.parseAlterStoragePolicy(startLoc) + default: + return p.unsupported("ALTER STORAGE") + } +} + +// parseDropStorage dispatches on VAULT|POLICY after STORAGE. +// DROP and STORAGE have already been consumed; cur is VAULT or POLICY. +func (p *Parser) parseDropStorage(startLoc ast.Loc) (ast.Node, error) { + switch p.cur.Kind { + case kwVAULT: + return p.parseDropStorageVault(startLoc) + case kwPOLICY: + return p.parseDropStoragePolicy(startLoc) + default: + return p.unsupported("DROP STORAGE") + } +} diff --git a/doris/parser/storage_test.go b/doris/parser/storage_test.go new file mode 100644 index 00000000..14e60519 --- /dev/null +++ b/doris/parser/storage_test.go @@ -0,0 +1,882 @@ +package parser + +import ( + "testing" + + "github.com/bytebase/omni/doris/ast" +) + +// --------------------------------------------------------------------------- +// CREATE STORAGE VAULT +// --------------------------------------------------------------------------- + +func TestCreateStorageVault_Basic(t *testing.T) { + sql := `CREATE STORAGE VAULT hdfs_vault PROPERTIES ( + "type" = "hdfs", + "fs.defaultFS" = "hdfs://127.0.0.1:8020" + )` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if len(file.Stmts) != 1 { + t.Fatalf("expected 1 stmt, got %d", len(file.Stmts)) + } + stmt, ok := file.Stmts[0].(*ast.CreateStorageVaultStmt) + if !ok { + t.Fatalf("expected *ast.CreateStorageVaultStmt, got %T", file.Stmts[0]) + } + if stmt.Name != "hdfs_vault" { + t.Errorf("Name = %q, want %q", stmt.Name, "hdfs_vault") + } + if stmt.IfNotExists { + t.Error("IfNotExists should be false") + } + if len(stmt.Properties) != 2 { + t.Fatalf("expected 2 properties, got %d", len(stmt.Properties)) + } + if stmt.Properties[0].Key != "type" { + t.Errorf("Properties[0].Key = %q, want %q", stmt.Properties[0].Key, "type") + } + if stmt.Properties[0].Value != "hdfs" { + t.Errorf("Properties[0].Value = %q, want %q", stmt.Properties[0].Value, "hdfs") + } +} + +func TestCreateStorageVault_IfNotExists(t *testing.T) { + sql := `CREATE STORAGE VAULT IF NOT EXISTS s3_vault PROPERTIES ( + "type" = "S3", + "s3.endpoint" = "s3.us-east-1.amazonaws.com", + "s3.access_key" = "xxxxxx", + "s3.secret_key" = "xxxxxx", + "s3.region" = "us-east-1", + "s3.root.path" = "prefix", + "s3.bucket" = "mybucket", + "provider" = "S3", + "use_path_style" = "false" + )` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt := file.Stmts[0].(*ast.CreateStorageVaultStmt) + if !stmt.IfNotExists { + t.Error("IfNotExists should be true") + } + if stmt.Name != "s3_vault" { + t.Errorf("Name = %q, want %q", stmt.Name, "s3_vault") + } + if len(stmt.Properties) != 9 { + t.Fatalf("expected 9 properties, got %d", len(stmt.Properties)) + } +} + +func TestCreateStorageVault_LegacyCorpus_HDFS(t *testing.T) { + sql := `CREATE STORAGE VAULT IF NOT EXISTS hdfs_vault_demo PROPERTIES ( + "type" = "hdfs", + "fs.defaultFS" = "hdfs://127.0.0.1:8020", + "path_prefix" = "big/data", + "hadoop.username" = "user", + "hadoop.security.authentication" = "kerberos", + "hadoop.kerberos.principal" = "hadoop/127.0.0.1@XXX", + "hadoop.kerberos.keytab" = "/etc/emr.keytab" + )` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt := file.Stmts[0].(*ast.CreateStorageVaultStmt) + if stmt.Name != "hdfs_vault_demo" { + t.Errorf("Name = %q, want %q", stmt.Name, "hdfs_vault_demo") + } + if len(stmt.Properties) != 7 { + t.Fatalf("expected 7 properties, got %d", len(stmt.Properties)) + } +} + +func TestCreateStorageVault_Tag(t *testing.T) { + file, errs := Parse(`CREATE STORAGE VAULT v PROPERTIES("type"="S3")`) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_CreateStorageVaultStmt { + t.Errorf("Tag() = %v, want T_CreateStorageVaultStmt", file.Stmts[0].Tag()) + } +} + +// --------------------------------------------------------------------------- +// ALTER STORAGE VAULT +// --------------------------------------------------------------------------- + +func TestAlterStorageVault_Basic(t *testing.T) { + sql := `ALTER STORAGE VAULT old_vault_name PROPERTIES ( + "type" = "S3", + "VAULT_NAME" = "new_vault_name", + "s3.access_key" = "new_ak" + )` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if len(file.Stmts) != 1 { + t.Fatalf("expected 1 stmt, got %d", len(file.Stmts)) + } + stmt, ok := file.Stmts[0].(*ast.AlterStorageVaultStmt) + if !ok { + t.Fatalf("expected *ast.AlterStorageVaultStmt, got %T", file.Stmts[0]) + } + if stmt.Name != "old_vault_name" { + t.Errorf("Name = %q, want %q", stmt.Name, "old_vault_name") + } + if len(stmt.Properties) != 3 { + t.Fatalf("expected 3 properties, got %d", len(stmt.Properties)) + } + if stmt.Properties[1].Key != "VAULT_NAME" { + t.Errorf("Properties[1].Key = %q, want %q", stmt.Properties[1].Key, "VAULT_NAME") + } + if stmt.Properties[1].Value != "new_vault_name" { + t.Errorf("Properties[1].Value = %q, want %q", stmt.Properties[1].Value, "new_vault_name") + } +} + +func TestAlterStorageVault_Tag(t *testing.T) { + file, errs := Parse(`ALTER STORAGE VAULT v PROPERTIES("type"="S3")`) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_AlterStorageVaultStmt { + t.Errorf("Tag() = %v, want T_AlterStorageVaultStmt", file.Stmts[0].Tag()) + } +} + +// --------------------------------------------------------------------------- +// DROP STORAGE VAULT +// --------------------------------------------------------------------------- + +func TestDropStorageVault_Basic(t *testing.T) { + file, errs := Parse("DROP STORAGE VAULT my_vault") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt, ok := file.Stmts[0].(*ast.DropStorageVaultStmt) + if !ok { + t.Fatalf("expected *ast.DropStorageVaultStmt, got %T", file.Stmts[0]) + } + if stmt.Name != "my_vault" { + t.Errorf("Name = %q, want %q", stmt.Name, "my_vault") + } + if stmt.IfExists { + t.Error("IfExists should be false") + } +} + +func TestDropStorageVault_IfExists(t *testing.T) { + file, errs := Parse("DROP STORAGE VAULT IF EXISTS my_vault") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt := file.Stmts[0].(*ast.DropStorageVaultStmt) + if !stmt.IfExists { + t.Error("IfExists should be true") + } + if stmt.Name != "my_vault" { + t.Errorf("Name = %q, want %q", stmt.Name, "my_vault") + } +} + +func TestDropStorageVault_Tag(t *testing.T) { + file, errs := Parse("DROP STORAGE VAULT v") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_DropStorageVaultStmt { + t.Errorf("Tag() = %v, want T_DropStorageVaultStmt", file.Stmts[0].Tag()) + } +} + +// --------------------------------------------------------------------------- +// SET / UNSET DEFAULT STORAGE VAULT +// --------------------------------------------------------------------------- + +func TestSetDefaultStorageVault(t *testing.T) { + file, errs := Parse("SET DEFAULT STORAGE VAULT my_vault") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if len(file.Stmts) != 1 { + t.Fatalf("expected 1 stmt, got %d", len(file.Stmts)) + } + stmt, ok := file.Stmts[0].(*ast.SetDefaultStorageVaultStmt) + if !ok { + t.Fatalf("expected *ast.SetDefaultStorageVaultStmt, got %T", file.Stmts[0]) + } + if stmt.Name != "my_vault" { + t.Errorf("Name = %q, want %q", stmt.Name, "my_vault") + } +} + +func TestSetDefaultStorageVault_Tag(t *testing.T) { + file, errs := Parse("SET DEFAULT STORAGE VAULT v") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_SetDefaultStorageVaultStmt { + t.Errorf("Tag() = %v, want T_SetDefaultStorageVaultStmt", file.Stmts[0].Tag()) + } +} + +func TestUnsetDefaultStorageVault(t *testing.T) { + file, errs := Parse("UNSET DEFAULT STORAGE VAULT") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if len(file.Stmts) != 1 { + t.Fatalf("expected 1 stmt, got %d", len(file.Stmts)) + } + _, ok := file.Stmts[0].(*ast.UnsetDefaultStorageVaultStmt) + if !ok { + t.Fatalf("expected *ast.UnsetDefaultStorageVaultStmt, got %T", file.Stmts[0]) + } +} + +func TestUnsetDefaultStorageVault_Tag(t *testing.T) { + file, errs := Parse("UNSET DEFAULT STORAGE VAULT") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_UnsetDefaultStorageVaultStmt { + t.Errorf("Tag() = %v, want T_UnsetDefaultStorageVaultStmt", file.Stmts[0].Tag()) + } +} + +// --------------------------------------------------------------------------- +// CREATE STORAGE POLICY +// --------------------------------------------------------------------------- + +func TestCreateStoragePolicy_Basic(t *testing.T) { + sql := `CREATE STORAGE POLICY testPolicy PROPERTIES( + "storage_resource" = "s3", + "cooldown_datetime" = "2022-06-08 00:00:00" + )` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if len(file.Stmts) != 1 { + t.Fatalf("expected 1 stmt, got %d", len(file.Stmts)) + } + stmt, ok := file.Stmts[0].(*ast.CreateStoragePolicyStmt) + if !ok { + t.Fatalf("expected *ast.CreateStoragePolicyStmt, got %T", file.Stmts[0]) + } + if stmt.Name != "testPolicy" { + t.Errorf("Name = %q, want %q", stmt.Name, "testPolicy") + } + if stmt.IfNotExists { + t.Error("IfNotExists should be false") + } + if len(stmt.Properties) != 2 { + t.Fatalf("expected 2 properties, got %d", len(stmt.Properties)) + } + if stmt.Properties[0].Key != "storage_resource" { + t.Errorf("Properties[0].Key = %q, want %q", stmt.Properties[0].Key, "storage_resource") + } +} + +func TestCreateStoragePolicy_CooldownTTL(t *testing.T) { + sql := `CREATE STORAGE POLICY testPolicy PROPERTIES( + "storage_resource" = "s3", + "cooldown_ttl" = "1d" + )` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt := file.Stmts[0].(*ast.CreateStoragePolicyStmt) + if stmt.Name != "testPolicy" { + t.Errorf("Name = %q, want %q", stmt.Name, "testPolicy") + } + if len(stmt.Properties) != 2 { + t.Fatalf("expected 2 properties, got %d", len(stmt.Properties)) + } + if stmt.Properties[1].Key != "cooldown_ttl" { + t.Errorf("Properties[1].Key = %q, want %q", stmt.Properties[1].Key, "cooldown_ttl") + } +} + +func TestCreateStoragePolicy_IfNotExists(t *testing.T) { + sql := `CREATE STORAGE POLICY IF NOT EXISTS myPolicy PROPERTIES("storage_resource"="s3")` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt := file.Stmts[0].(*ast.CreateStoragePolicyStmt) + if !stmt.IfNotExists { + t.Error("IfNotExists should be true") + } +} + +func TestCreateStoragePolicy_Tag(t *testing.T) { + file, errs := Parse(`CREATE STORAGE POLICY p PROPERTIES("storage_resource"="s3")`) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_CreateStoragePolicyStmt { + t.Errorf("Tag() = %v, want T_CreateStoragePolicyStmt", file.Stmts[0].Tag()) + } +} + +// --------------------------------------------------------------------------- +// ALTER STORAGE POLICY +// --------------------------------------------------------------------------- + +func TestAlterStoragePolicy_Basic(t *testing.T) { + sql := `ALTER STORAGE POLICY testPolicy PROPERTIES("cooldown_ttl"="2d")` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt, ok := file.Stmts[0].(*ast.AlterStoragePolicyStmt) + if !ok { + t.Fatalf("expected *ast.AlterStoragePolicyStmt, got %T", file.Stmts[0]) + } + if stmt.Name != "testPolicy" { + t.Errorf("Name = %q, want %q", stmt.Name, "testPolicy") + } + if len(stmt.Properties) != 1 { + t.Fatalf("expected 1 property, got %d", len(stmt.Properties)) + } +} + +func TestAlterStoragePolicy_Tag(t *testing.T) { + file, errs := Parse(`ALTER STORAGE POLICY p PROPERTIES("cooldown_ttl"="1d")`) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_AlterStoragePolicyStmt { + t.Errorf("Tag() = %v, want T_AlterStoragePolicyStmt", file.Stmts[0].Tag()) + } +} + +// --------------------------------------------------------------------------- +// DROP STORAGE POLICY +// --------------------------------------------------------------------------- + +func TestDropStoragePolicy_Basic(t *testing.T) { + file, errs := Parse("DROP STORAGE POLICY policy1") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt, ok := file.Stmts[0].(*ast.DropStoragePolicyStmt) + if !ok { + t.Fatalf("expected *ast.DropStoragePolicyStmt, got %T", file.Stmts[0]) + } + if stmt.Name != "policy1" { + t.Errorf("Name = %q, want %q", stmt.Name, "policy1") + } + if stmt.IfExists { + t.Error("IfExists should be false") + } +} + +func TestDropStoragePolicy_IfExists(t *testing.T) { + file, errs := Parse("DROP STORAGE POLICY IF EXISTS policy1") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt := file.Stmts[0].(*ast.DropStoragePolicyStmt) + if !stmt.IfExists { + t.Error("IfExists should be true") + } +} + +func TestDropStoragePolicy_Tag(t *testing.T) { + file, errs := Parse("DROP STORAGE POLICY p") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_DropStoragePolicyStmt { + t.Errorf("Tag() = %v, want T_DropStoragePolicyStmt", file.Stmts[0].Tag()) + } +} + +// --------------------------------------------------------------------------- +// CREATE REPOSITORY +// --------------------------------------------------------------------------- + +func TestCreateRepository_S3(t *testing.T) { + sql := `CREATE REPOSITORY s3_repo + WITH S3 + ON LOCATION "s3://bucket/path" + PROPERTIES( + "s3.endpoint" = "s3.us-east-1.amazonaws.com", + "s3.access_key" = "xxxxxx", + "s3.secret_key" = "xxxxxx", + "s3.region" = "us-east-1" + )` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt, ok := file.Stmts[0].(*ast.CreateRepositoryStmt) + if !ok { + t.Fatalf("expected *ast.CreateRepositoryStmt, got %T", file.Stmts[0]) + } + if stmt.Name != "s3_repo" { + t.Errorf("Name = %q, want %q", stmt.Name, "s3_repo") + } + if stmt.ReadOnly { + t.Error("ReadOnly should be false") + } + if stmt.Type != "S3" { + t.Errorf("Type = %q, want %q", stmt.Type, "S3") + } + if stmt.BrokerName != "" { + t.Errorf("BrokerName = %q, want empty", stmt.BrokerName) + } + if len(stmt.Properties) != 4 { + t.Fatalf("expected 4 properties, got %d", len(stmt.Properties)) + } +} + +func TestCreateRepository_HDFS(t *testing.T) { + sql := `CREATE REPOSITORY hdfs_repo + WITH HDFS + ON LOCATION "hdfs://namenode:8020/path" + PROPERTIES( + "hadoop.username" = "user" + )` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt := file.Stmts[0].(*ast.CreateRepositoryStmt) + if stmt.Name != "hdfs_repo" { + t.Errorf("Name = %q, want %q", stmt.Name, "hdfs_repo") + } + if stmt.Type != "HDFS" { + t.Errorf("Type = %q, want %q", stmt.Type, "HDFS") + } +} + +func TestCreateRepository_Broker(t *testing.T) { + sql := `CREATE REPOSITORY broker_repo + WITH BROKER my_broker + ON LOCATION "hdfs://namenode:8020/backup" + PROPERTIES( + "username" = "hdfsuser", + "password" = "secret" + )` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt := file.Stmts[0].(*ast.CreateRepositoryStmt) + if stmt.Type != "BROKER" { + t.Errorf("Type = %q, want %q", stmt.Type, "BROKER") + } + if stmt.BrokerName != "my_broker" { + t.Errorf("BrokerName = %q, want %q", stmt.BrokerName, "my_broker") + } +} + +func TestCreateRepository_ReadOnly(t *testing.T) { + sql := `CREATE READ ONLY REPOSITORY ro_repo + WITH S3 + ON LOCATION "s3://bucket/path" + PROPERTIES("s3.endpoint"="s3.amazonaws.com","s3.access_key"="ak","s3.secret_key"="sk","s3.region"="us-east-1")` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt := file.Stmts[0].(*ast.CreateRepositoryStmt) + if !stmt.ReadOnly { + t.Error("ReadOnly should be true") + } + if stmt.Name != "ro_repo" { + t.Errorf("Name = %q, want %q", stmt.Name, "ro_repo") + } +} + +func TestCreateRepository_Tag(t *testing.T) { + sql := `CREATE REPOSITORY r WITH S3 ON LOCATION "s3://b/p" PROPERTIES("s3.endpoint"="ep","s3.access_key"="ak","s3.secret_key"="sk","s3.region"="us-east-1")` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_CreateRepositoryStmt { + t.Errorf("Tag() = %v, want T_CreateRepositoryStmt", file.Stmts[0].Tag()) + } +} + +// --------------------------------------------------------------------------- +// ALTER REPOSITORY +// --------------------------------------------------------------------------- + +func TestAlterRepository_Basic(t *testing.T) { + sql := `ALTER REPOSITORY my_repo PROPERTIES("s3.access_key"="new_key","s3.secret_key"="new_secret")` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt, ok := file.Stmts[0].(*ast.AlterRepositoryStmt) + if !ok { + t.Fatalf("expected *ast.AlterRepositoryStmt, got %T", file.Stmts[0]) + } + if stmt.Name != "my_repo" { + t.Errorf("Name = %q, want %q", stmt.Name, "my_repo") + } + if len(stmt.Properties) != 2 { + t.Fatalf("expected 2 properties, got %d", len(stmt.Properties)) + } +} + +func TestAlterRepository_Tag(t *testing.T) { + file, errs := Parse(`ALTER REPOSITORY r PROPERTIES("s3.access_key"="ak")`) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_AlterRepositoryStmt { + t.Errorf("Tag() = %v, want T_AlterRepositoryStmt", file.Stmts[0].Tag()) + } +} + +// --------------------------------------------------------------------------- +// DROP REPOSITORY +// --------------------------------------------------------------------------- + +func TestDropRepository_Basic(t *testing.T) { + file, errs := Parse("DROP REPOSITORY my_repo") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt, ok := file.Stmts[0].(*ast.DropRepositoryStmt) + if !ok { + t.Fatalf("expected *ast.DropRepositoryStmt, got %T", file.Stmts[0]) + } + if stmt.Name != "my_repo" { + t.Errorf("Name = %q, want %q", stmt.Name, "my_repo") + } +} + +func TestDropRepository_Tag(t *testing.T) { + file, errs := Parse("DROP REPOSITORY r") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_DropRepositoryStmt { + t.Errorf("Tag() = %v, want T_DropRepositoryStmt", file.Stmts[0].Tag()) + } +} + +// --------------------------------------------------------------------------- +// CREATE STAGE +// --------------------------------------------------------------------------- + +func TestCreateStage_Basic(t *testing.T) { + sql := `CREATE STAGE my_stage PROPERTIES( + "endpoint" = "cos.ap-guangzhou.myqcloud.com", + "access_key_id" = "xxxxxx", + "access_key_secret" = "xxxxxx", + "bucket" = "mybucket", + "path" = "mypath" + )` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt, ok := file.Stmts[0].(*ast.CreateStageStmt) + if !ok { + t.Fatalf("expected *ast.CreateStageStmt, got %T", file.Stmts[0]) + } + if stmt.Name != "my_stage" { + t.Errorf("Name = %q, want %q", stmt.Name, "my_stage") + } + if stmt.IfNotExists { + t.Error("IfNotExists should be false") + } + if len(stmt.Properties) != 5 { + t.Fatalf("expected 5 properties, got %d", len(stmt.Properties)) + } +} + +func TestCreateStage_IfNotExists(t *testing.T) { + sql := `CREATE STAGE IF NOT EXISTS my_stage PROPERTIES("endpoint"="ep","access_key_id"="ak","access_key_secret"="sk","bucket"="b","path"="p")` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt := file.Stmts[0].(*ast.CreateStageStmt) + if !stmt.IfNotExists { + t.Error("IfNotExists should be true") + } +} + +func TestCreateStage_Tag(t *testing.T) { + sql := `CREATE STAGE s PROPERTIES("endpoint"="ep","access_key_id"="ak","access_key_secret"="sk","bucket"="b","path"="p")` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_CreateStageStmt { + t.Errorf("Tag() = %v, want T_CreateStageStmt", file.Stmts[0].Tag()) + } +} + +// --------------------------------------------------------------------------- +// DROP STAGE +// --------------------------------------------------------------------------- + +func TestDropStage_Basic(t *testing.T) { + file, errs := Parse("DROP STAGE my_stage") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt, ok := file.Stmts[0].(*ast.DropStageStmt) + if !ok { + t.Fatalf("expected *ast.DropStageStmt, got %T", file.Stmts[0]) + } + if stmt.Name != "my_stage" { + t.Errorf("Name = %q, want %q", stmt.Name, "my_stage") + } + if stmt.IfExists { + t.Error("IfExists should be false") + } +} + +func TestDropStage_IfExists(t *testing.T) { + file, errs := Parse("DROP STAGE IF EXISTS my_stage") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt := file.Stmts[0].(*ast.DropStageStmt) + if !stmt.IfExists { + t.Error("IfExists should be true") + } + if stmt.Name != "my_stage" { + t.Errorf("Name = %q, want %q", stmt.Name, "my_stage") + } +} + +func TestDropStage_Tag(t *testing.T) { + file, errs := Parse("DROP STAGE s") + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_DropStageStmt { + t.Errorf("Tag() = %v, want T_DropStageStmt", file.Stmts[0].Tag()) + } +} + +// --------------------------------------------------------------------------- +// CREATE FILE +// --------------------------------------------------------------------------- + +func TestCreateFile_Basic(t *testing.T) { + sql := `CREATE FILE "ca.pem" PROPERTIES("url"="https://example.com/ca.pem","type"="ca")` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt, ok := file.Stmts[0].(*ast.CreateFileStmt) + if !ok { + t.Fatalf("expected *ast.CreateFileStmt, got %T", file.Stmts[0]) + } + if stmt.Name != "ca.pem" { + t.Errorf("Name = %q, want %q", stmt.Name, "ca.pem") + } + if stmt.Database != "" { + t.Errorf("Database = %q, want empty", stmt.Database) + } + if len(stmt.Properties) != 2 { + t.Fatalf("expected 2 properties, got %d", len(stmt.Properties)) + } +} + +func TestCreateFile_WithDatabase(t *testing.T) { + sql := `CREATE FILE "my_cert.pem" IN mydb PROPERTIES("url"="https://example.com/cert.pem","type"="ca")` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt := file.Stmts[0].(*ast.CreateFileStmt) + if stmt.Name != "my_cert.pem" { + t.Errorf("Name = %q, want %q", stmt.Name, "my_cert.pem") + } + if stmt.Database != "mydb" { + t.Errorf("Database = %q, want %q", stmt.Database, "mydb") + } +} + +func TestCreateFile_Tag(t *testing.T) { + sql := `CREATE FILE "f.pem" PROPERTIES("url"="https://x.com/f.pem","type"="ca")` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_CreateFileStmt { + t.Errorf("Tag() = %v, want T_CreateFileStmt", file.Stmts[0].Tag()) + } +} + +// --------------------------------------------------------------------------- +// DROP FILE +// --------------------------------------------------------------------------- + +func TestDropFile_Basic(t *testing.T) { + sql := `DROP FILE "ca.pem" PROPERTIES("type"="ca")` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt, ok := file.Stmts[0].(*ast.DropFileStmt) + if !ok { + t.Fatalf("expected *ast.DropFileStmt, got %T", file.Stmts[0]) + } + if stmt.Name != "ca.pem" { + t.Errorf("Name = %q, want %q", stmt.Name, "ca.pem") + } + if stmt.Database != "" { + t.Errorf("Database = %q, want empty", stmt.Database) + } + if len(stmt.Properties) != 1 { + t.Fatalf("expected 1 property, got %d", len(stmt.Properties)) + } +} + +func TestDropFile_WithDatabase(t *testing.T) { + sql := `DROP FILE "ca.pem" FROM mydb PROPERTIES("type"="ca")` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + stmt := file.Stmts[0].(*ast.DropFileStmt) + if stmt.Database != "mydb" { + t.Errorf("Database = %q, want %q", stmt.Database, "mydb") + } +} + +func TestDropFile_Tag(t *testing.T) { + sql := `DROP FILE "f.pem" PROPERTIES("type"="ca")` + file, errs := Parse(sql) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if file.Stmts[0].Tag() != ast.T_DropFileStmt { + t.Errorf("Tag() = %v, want T_DropFileStmt", file.Stmts[0].Tag()) + } +} + +// --------------------------------------------------------------------------- +// Legacy corpus round-trip +// --------------------------------------------------------------------------- + +func TestLegacyCorpus_StoragePolicy(t *testing.T) { + // From doris/parser/testdata/legacy/cluster_storage_policy.sql + tests := []struct { + sql string + wantType string + }{ + { + `CREATE STORAGE POLICY testPolicy PROPERTIES( + "storage_resource" = "s3", + "cooldown_datetime" = "2022-06-08 00:00:00" + )`, + "*ast.CreateStoragePolicyStmt", + }, + { + `CREATE STORAGE POLICY testPolicy PROPERTIES( + "storage_resource" = "s3", + "cooldown_ttl" = "1d" + )`, + "*ast.CreateStoragePolicyStmt", + }, + { + `DROP STORAGE POLICY policy1`, + "*ast.DropStoragePolicyStmt", + }, + } + + for _, tt := range tests { + file, errs := Parse(tt.sql) + if len(errs) != 0 { + t.Errorf("sql %q: unexpected errors: %v", tt.sql, errs) + continue + } + if len(file.Stmts) != 1 { + t.Errorf("sql %q: expected 1 stmt, got %d", tt.sql, len(file.Stmts)) + } + } +} + +func TestLegacyCorpus_StorageVault(t *testing.T) { + // From doris/parser/testdata/legacy/cluster_storage_vault.sql + tests := []string{ + `CREATE STORAGE VAULT IF NOT EXISTS hdfs_vault_demo PROPERTIES ( + "type" = "hdfs", + "fs.defaultFS" = "hdfs://127.0.0.1:8020", + "path_prefix" = "big/data", + "hadoop.username" = "user", + "hadoop.security.authentication" = "kerberos", + "hadoop.kerberos.principal" = "hadoop/127.0.0.1@XXX", + "hadoop.kerberos.keytab" = "/etc/emr.keytab" + )`, + `CREATE STORAGE VAULT IF NOT EXISTS oss_demo_vault PROPERTIES ( + "type" = "S3", + "s3.endpoint" = "oss-cn-beijing.aliyuncs.com", + "s3.access_key" = "xxxxxx", + "s3.secret_key" = "xxxxxx", + "s3.region" = "cn-beijing", + "s3.root.path" = "oss_demo_vault_prefix", + "s3.bucket" = "xxxxxx", + "provider" = "OSS", + "use_path_style" = "false" + )`, + `ALTER STORAGE VAULT old_vault_name PROPERTIES ( + "type" = "S3", + "VAULT_NAME" = "new_vault_name", + "s3.access_key" = "new_ak" + )`, + } + + for _, sql := range tests { + file, errs := Parse(sql) + if len(errs) != 0 { + t.Errorf("sql:\n%s\nunexpected errors: %v", sql, errs) + continue + } + if len(file.Stmts) != 1 { + t.Errorf("sql:\n%s\nexpected 1 stmt, got %d", sql, len(file.Stmts)) + } + } +} + +// --------------------------------------------------------------------------- +// Multi-statement round-trip +// --------------------------------------------------------------------------- + +func TestStorage_MultiStatement(t *testing.T) { + input := `CREATE STORAGE VAULT v PROPERTIES("type"="S3"); +DROP STORAGE VAULT v; +CREATE STORAGE POLICY p PROPERTIES("storage_resource"="s3"); +DROP STORAGE POLICY p` + file, errs := Parse(input) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if len(file.Stmts) != 4 { + t.Fatalf("expected 4 stmts, got %d", len(file.Stmts)) + } + if _, ok := file.Stmts[0].(*ast.CreateStorageVaultStmt); !ok { + t.Errorf("Stmts[0]: expected CreateStorageVaultStmt, got %T", file.Stmts[0]) + } + if _, ok := file.Stmts[1].(*ast.DropStorageVaultStmt); !ok { + t.Errorf("Stmts[1]: expected DropStorageVaultStmt, got %T", file.Stmts[1]) + } + if _, ok := file.Stmts[2].(*ast.CreateStoragePolicyStmt); !ok { + t.Errorf("Stmts[2]: expected CreateStoragePolicyStmt, got %T", file.Stmts[2]) + } + if _, ok := file.Stmts[3].(*ast.DropStoragePolicyStmt); !ok { + t.Errorf("Stmts[3]: expected DropStoragePolicyStmt, got %T", file.Stmts[3]) + } +}