Incremental AST and ValueType

Author: Your Name

lib/symboldatabase.cpp

@@ -2108,9 +2108,17 @@ void SymbolDatabase::addSymbolsForNewTokenRanges(const std::vector<std::pair<Tok
         }
     }
 
-    // function/type/enumerator pointers - these only set pointers that are not set, so
-    // the old tokens are not affected
-    createSymbolDatabaseSetFunctionPointers(true);
+    // set the definition and declaration pointers of the new functions. The call
+    // pointers are updated afterwards by updateFunctionAndVariablePointers().
+    for (const auto& f : newFunctions) {
+        if (f.first->tokenDef)
+            const_cast<Token *>(f.first->tokenDef)->function(f.first);
+        if (f.first->token)
+            const_cast<Token *>(f.first->token)->function(f.first);
+    }
+
+    // type/enumerator pointers - these only set pointers that are not set, so the old
+    // tokens are not affected
     createSymbolDatabaseSetTypePointers();
     createSymbolDatabaseSetSmartPointerType();
     createSymbolDatabaseEnums();
@@ -8043,15 +8051,15 @@ static int getIntegerConstantMacroWidth(const Token* tok) {
     return intnum;
 }
 
-void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *tokens)
+void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *tokens, const Token *endToken)
 {
     if (!tokens)
         tokens = mTokenizer.list.front();
 
-    for (Token *tok = tokens; tok; tok = tok->next())
+    for (Token *tok = tokens; tok && tok != endToken; tok = tok->next())
         tok->setValueType(nullptr);
 
-    for (Token *tok = tokens; tok; tok = tok->next()) {
+    for (Token *tok = tokens; tok && tok != endToken; tok = tok->next()) {
         if (tok->isNumber()) {
             if (MathLib::isFloat(tok->str())) {
                 ValueType::Type type = ValueType::Type::DOUBLE;
@@ -8486,7 +8494,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
     }
 
     if (reportDebugWarnings && mSettings.debugwarnings) {
-        for (Token *tok = tokens; tok; tok = tok->next()) {
+        for (Token *tok = tokens; tok && tok != endToken; tok = tok->next()) {
             if (tok->str() == "auto" && !tok->valueType()) {
                 if (Token::Match(tok->next(), "%name% ; %name% = [") && isLambdaCaptureList(tok->tokAt(5)))
                     continue;
@@ -8497,13 +8505,23 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
         }
     }
 
+    // when only a range was set the caller is responsible for updating the pointers
+    if (endToken)
+        return;
+
     // Update functions with new type information.
     createSymbolDatabaseSetFunctionPointers(false);
 
     // Update auto variables with new type information.
     createSymbolDatabaseSetVariablePointers();
 }
 
+void SymbolDatabase::updateFunctionAndVariablePointers()
+{
+    createSymbolDatabaseSetFunctionPointers(false);
+    createSymbolDatabaseSetVariablePointers();
+}
+
 ValueType ValueType::parseDecl(const Token *type, const Settings &settings)
 {
     ValueType vt;

lib/symboldatabase.h

@@ -1416,8 +1416,14 @@ class CPPCHECKLIB SymbolDatabase {
      */
     void validate() const;
 
-    /** Set valuetype in provided tokenlist */
-    void setValueTypeInTokenList(bool reportDebugWarnings, Token *tokens=nullptr);
+    /** Set valuetype in provided tokenlist. When endToken (exclusive) is given, only
+     * that token range is set and the caller is responsible for updating the function
+     * and variable pointers (updateFunctionAndVariablePointers()). */
+    void setValueTypeInTokenList(bool reportDebugWarnings, Token *tokens=nullptr, const Token *endToken=nullptr);
+
+    /** Update the function pointers of calls and the variable pointers in the whole
+     * token list, e.g. after tokens were added or renamed. */
+    void updateFunctionAndVariablePointers();
 
     /**
      * Calculates sizeof value for given type.

lib/templatesimplifier.cpp

@@ -1335,6 +1335,7 @@ void TemplateSimplifier::deduceFunctionTemplateArguments(Token* tok,
     }
     tok->insertToken("<");
     rememberNewTokens(tok->next(), closingBracket);
+    rememberDirtyCallSite(tok);
     mTypeDeductionsMade = true;
     mChanged = true;
 }
@@ -3934,6 +3935,7 @@ void TemplateSimplifier::replaceTemplateUsage(
 
         const Token * const nameTok1 = nameTok;
         nameTok->str(newName);
+        rememberDirtyCallSite(nameTok);
 
         // matching template usage => replace tokens..
         // Foo < int >  =>  Foo<int>
@@ -4631,6 +4633,7 @@ bool TemplateSimplifier::simplifyTemplatesUsingTypeInformation(std::time_t maxti
     mPendingTypeDeductions = false;
     mNewTokenRanges.clear();
     mConvertedSpecializations.clear();
+    mDirtyCallSites.clear();
     mIncrementalUpdatePossible = true;
     // start from a clean state - the state left behind by the previous
     // simplifyTemplates() call refers to a token list that has changed since
@@ -4666,6 +4669,14 @@ void TemplateSimplifier::rememberNewTokens(Token* first, Token* last)
     mNewTokenRanges.emplace_back(first, last);
 }
 
+void TemplateSimplifier::rememberDirtyCallSite(Token* nameTok)
+{
+    if (!mUseTypeInformation || !nameTok)
+        return;
+    if (std::find(mDirtyCallSites.cbegin(), mDirtyCallSites.cend(), nameTok) == mDirtyCallSites.cend())
+        mDirtyCallSites.push_back(nameTok);
+}
+
 bool TemplateSimplifier::removeDeferredTemplateDeclarations(SymbolDatabase* symbolDatabase)
 {
     // the declarations could have been removed by a later simplification - make sure
@@ -4676,14 +4687,30 @@ bool TemplateSimplifier::removeDeferredTemplateDeclarations(SymbolDatabase* symb
     });
     mDeferredRemovals.clear();
 
+    std::vector<std::pair<Token*, const Token*>> ranges;
+    for (Token* declTok : declarations) {
+        const Token* end = findTemplateDeclarationEnd(declTok);
+        if (end)
+            ranges.emplace_back(declTok, end);
+    }
+
+    // dirty call sites inside the removed declarations are gone
+    if (!mDirtyCallSites.empty() && !ranges.empty()) {
+        std::set<const Token*> removedTokens;
+        for (const auto& range : ranges) {
+            for (const Token* tok = range.first; tok; tok = tok->next()) {
+                removedTokens.insert(tok);
+                if (tok == range.second)
+                    break;
+            }
+        }
+        mDirtyCallSites.erase(std::remove_if(mDirtyCallSites.begin(), mDirtyCallSites.end(), [&](const Token* site) {
+            return removedTokens.count(site) != 0;
+        }), mDirtyCallSites.end());
+    }
+
     if (symbolDatabase) {
         // remove the symbols before the tokens are removed
-        std::vector<std::pair<Token*, const Token*>> ranges;
-        for (Token* declTok : declarations) {
-            const Token* end = findTemplateDeclarationEnd(declTok);
-            if (end)
-                ranges.emplace_back(declTok, end);
-        }
         symbolDatabase->removeSymbolsInTokenRanges(ranges);
     }
 

lib/templatesimplifier.h

@@ -358,10 +358,19 @@ class CPPCHECKLIB TemplateSimplifier {
         return mConvertedSpecializations;
     }
 
+    /** name tokens of the call sites that the last simplifyTemplatesUsingTypeInformation()
+     * call changed (deduced template arguments and qualifications were inserted, or the
+     * call was renamed to an instantiation) - the AST and the valuetypes of the
+     * statements around them must be recreated */
+    const std::vector<Token*>& dirtyCallSites() const {
+        return mDirtyCallSites;
+    }
+
     /** forget the recorded token changes, called when they have been consumed */
     void clearTokenChanges() {
         mNewTokenRanges.clear();
         mConvertedSpecializations.clear();
+        mDirtyCallSites.clear();
     }
 
     /** true when the token changes made by the last simplifyTemplatesUsingTypeInformation()
@@ -433,6 +442,10 @@ class CPPCHECKLIB TemplateSimplifier {
      * database can be updated incrementally */
     void rememberNewTokens(Token* first, Token* last);
 
+    /** remember a call site whose statement was changed during a type-information round
+     * so its AST and valuetypes can be recreated */
+    void rememberDirtyCallSite(Token* nameTok);
+
     /**
      * Get template instantiations
      */
@@ -605,6 +618,8 @@ class CPPCHECKLIB TemplateSimplifier {
     /** specializations that were turned into normal functions during type-information
      * rounds - their Function::templateDef must be cleared */
     std::vector<const Token*> mConvertedSpecializations;
+    /** call sites changed during type-information rounds (see dirtyCallSites()) */
+    std::vector<Token*> mDirtyCallSites;
     /** false when the current round made token changes that the incremental symbol
      * database update can not handle (class/variable/specialization instantiations,
      * type alias changes) */

lib/tokenize.cpp

@@ -4348,14 +4348,68 @@ void Tokenizer::rebuildTokenDataAfterTemplateSimplification()
 void Tokenizer::updateTokenDataAfterTemplateSimplification()
 {
     // update the supporting token information incrementally: the information of the
-    // unchanged tokens - variable ids, symbols, valuetypes - stays valid
+    // unchanged tokens - variable ids, symbols, AST, valuetypes - stays valid
     setVarId(/*incremental*/ true);
     createLinks2();   // adds the "<>" links of the new tokens
     Token::assignProgressValues(list.front());
     list.front()->assignIndexes();
-    list.clearAst();
-    list.createAst();
+
+    // The token regions whose AST and valuetypes must be recreated: the new tokens and
+    // the function bodies around the changed call sites.
+    bool rebuildAll = false;
+    // the regions (first and last token, inclusive) whose AST and valuetypes must be recreated
+    std::vector<std::pair<Token*, Token*>> regions = mTemplateSimplifier->newTokenRanges();
+    for (const Token* siteTok : mTemplateSimplifier->dirtyCallSites()) {
+        const Scope* scope = siteTok->scope();
+        if (!scope)
+            continue;   // a new token - covered by the new token ranges
+        // the outermost executable scope is the function body around the call
+        const Scope* functionScope = nullptr;
+        for (const Scope* s = scope; s; s = s->nestedIn) {
+            if (s->isExecutable())
+                functionScope = s;
+        }
+        if (!functionScope || !functionScope->bodyStart || !functionScope->bodyEnd ||
+            functionScope->bodyStart->astParent()) {
+            // no function body around the call (global initializer, lambda body that is
+            // part of an expression, ..) - recreate everything
+            rebuildAll = true;
+            break;
+        }
+        regions.emplace_back(const_cast<Token*>(functionScope->bodyStart), const_cast<Token*>(functionScope->bodyEnd));
+    }
+
+    if (!rebuildAll && !regions.empty()) {
+        // merge the overlapping regions - Token::index() is up to date
+        std::sort(regions.begin(), regions.end(), [](const std::pair<Token*, Token*>& lhs, const std::pair<Token*, Token*>& rhs) {
+            return lhs.first->index() < rhs.first->index();
+        });
+        std::vector<std::pair<Token*, Token*>> merged;
+        for (const auto& region : regions) {
+            if (!merged.empty() && region.first->index() <= merged.back().second->index()) {
+                if (region.second->index() > merged.back().second->index())
+                    merged.back().second = region.second;
+            } else
+                merged.push_back(region);
+        }
+        regions = std::move(merged);
+    }
+
+    if (rebuildAll) {
+        list.clearAst();
+        list.createAst();
+    } else {
+        for (const auto& region : regions) {
+            for (Token* tok = region.first; tok; tok = tok->next()) {
+                tok->clearAst();
+                if (tok == region.second)
+                    break;
+            }
+            list.createAst(region.first, region.second->next());
+        }
+    }
     list.validateAst(mSettings.debugnormal);
+
     for (const Token* nameTok : mTemplateSimplifier->convertedSpecializations()) {
         // the specialization became a normal function: its "template < >" tokens are gone
         if (nameTok->function())
@@ -4369,7 +4423,6 @@ void Tokenizer::updateTokenDataAfterTemplateSimplification()
             tok->tokType(Token::eName);
     }
     mSymbolDatabase->addSymbolsForNewTokenRanges(mTemplateSimplifier->newTokenRanges());
-    mTemplateSimplifier->clearTokenChanges();
     // safety net: every token needs a scope. A token that was added by a simplification
     // that is not covered by the new token ranges gets the scope of the preceding token.
     const Scope* lastScope = nullptr;
@@ -4381,8 +4434,19 @@ void Tokenizer::updateTokenDataAfterTemplateSimplification()
         } else if (lastScope)
             tok->scope(lastScope);
     }
-    mSymbolDatabase->setValueTypeInTokenList(false);
-    mSymbolDatabase->setValueTypeInTokenList(true);
+
+    // recreate the valuetypes of the changed regions
+    if (rebuildAll) {
+        mSymbolDatabase->setValueTypeInTokenList(false);
+        mSymbolDatabase->setValueTypeInTokenList(true);
+    } else {
+        mSymbolDatabase->updateFunctionAndVariablePointers();
+        for (const auto& region : regions) {
+            mSymbolDatabase->setValueTypeInTokenList(false, region.first, region.second->next());
+            mSymbolDatabase->setValueTypeInTokenList(true, region.first, region.second->next());
+        }
+    }
+    mTemplateSimplifier->clearTokenChanges();
     mSymbolDatabase->validate();
 }
 //---------------------------------------------------------------------------

lib/tokenlist.cpp

@@ -1904,13 +1904,18 @@ void TokenList::clearAst() const
 
 void TokenList::createAst() const
 {
-    for (Token *tok = mTokensFrontBack->front; tok; tok = tok ? tok->next() : nullptr) {
+    createAst(mTokensFrontBack->front, nullptr);
+}
+
+void TokenList::createAst(Token* startToken, const Token* endToken)
+{
+    for (Token *tok = startToken; tok && tok != endToken; tok = tok ? tok->next() : nullptr) {
         Token* const nextTok = createAstAtToken(tok);
         if (precedes(nextTok, tok))
             throw InternalError(tok, "Syntax Error: Infinite loop when creating AST.", InternalError::AST);
         tok = nextTok;
     }
-    for (Token *tok = mTokensFrontBack->front; tok; tok = tok ? tok->next() : nullptr) {
+    for (Token *tok = startToken; tok && tok != endToken; tok = tok ? tok->next() : nullptr) {
         if (tok->astParent())
             continue;
         if (!tok->astOperand1() && !tok->astOperand2())

lib/tokenlist.h

@@ -172,6 +172,13 @@ class CPPCHECKLIB TokenList {
      */
     void createAst() const;
 
+    /**
+     * Create the abstract syntax tree for the tokens in the given range. endToken is
+     * exclusive, nullptr means the end of the token list.
+     * @throws InternalError thrown if encountering an infinite loop in AST creation
+     */
+    static void createAst(Token* startToken, const Token* endToken);
+
     /**
      * Remove the abstract syntax tree from all tokens so createAst() can be called again,
      * for instance after the token list has been modified.