Skip to content

Commit

Permalink
[C++20] [Modules] Makes sure internal declaration won't be found by o…
Browse files Browse the repository at this point in the history 
…ther TU (#123059)

Close #61427

And this is also helpful to implement
#112294 partially.

The implementation strategy mimics
#122887. This patch split the
internal declarations from the general lookup table so that other TU
can't find the internal declarations.
  • Loading branch information
@ChuanqiXu9
ChuanqiXu9 authored Jan 17, 2025
1 parent 22637a8 commit fb2c9d9
Show file tree
Hide file tree
Showing 11 changed files with 327 additions and 115 deletions.
6 changes: 6 additions & 0 deletions clang/include/clang/Serialization/ASTBitCodes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -740,6 +740,8 @@ enum ASTRecordTypes {
CXX_ADDED_TEMPLATE_PARTIAL_SPECIALIZATION = 75,

UPDATE_MODULE_LOCAL_VISIBLE = 76,

UPDATE_TU_LOCAL_VISIBLE = 77,
};

/// Record types used within a source manager block.
Expand Down Expand Up @@ -1340,6 +1342,10 @@ enum DeclCode {
/// only visible from DeclContext in the same module.
DECL_CONTEXT_MODULE_LOCAL_VISIBLE,

/// A record that stores the set of declarations that are only visible
/// to the TU.
DECL_CONTEXT_TU_LOCAL_VISIBLE,

/// A LabelDecl record.
DECL_LABEL,

Expand Down
20 changes: 19 additions & 1 deletion clang/include/clang/Serialization/ASTReader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -528,6 +528,7 @@ class ASTReader
uint64_t LexicalOffset;
uint64_t VisibleOffset;
uint64_t ModuleLocalOffset;
uint64_t TULocalOffset;
};

using DelayedNamespaceOffsetMapTy =
Expand Down Expand Up @@ -640,6 +641,9 @@ class ASTReader
llvm::DenseMap<const DeclContext *,
serialization::reader::ModuleLocalLookupTable>
ModuleLocalLookups;
llvm::DenseMap<const DeclContext *,
serialization::reader::DeclContextLookupTable>
TULocalLookups;

using SpecLookupTableTy =
llvm::DenseMap<const Decl *,
Expand Down Expand Up @@ -670,6 +674,7 @@ class ASTReader
llvm::DenseMap<GlobalDeclID, DeclContextVisibleUpdates> PendingVisibleUpdates;
llvm::DenseMap<GlobalDeclID, DeclContextVisibleUpdates>
PendingModuleLocalVisibleUpdates;
llvm::DenseMap<GlobalDeclID, DeclContextVisibleUpdates> TULocalUpdates;

using SpecializationsUpdate = SmallVector<UpdateData, 1>;
using SpecializationsUpdateMap =
Expand Down Expand Up @@ -704,11 +709,17 @@ class ASTReader
llvm::BitstreamCursor &Cursor,
uint64_t Offset, DeclContext *DC);

enum class VisibleDeclContextStorageKind {
GenerallyVisible,
ModuleLocalVisible,
TULocalVisible,
};

/// Read the record that describes the visible contents of a DC.
bool ReadVisibleDeclContextStorage(ModuleFile &M,
llvm::BitstreamCursor &Cursor,
uint64_t Offset, GlobalDeclID ID,
bool IsModuleLocal);
VisibleDeclContextStorageKind VisibleKind);

bool ReadSpecializations(ModuleFile &M, llvm::BitstreamCursor &Cursor,
uint64_t Offset, Decl *D, bool IsPartial);
Expand Down Expand Up @@ -1148,6 +1159,10 @@ class ASTReader
unsigned NumModuleLocalVisibleDeclContexts = 0,
TotalModuleLocalVisibleDeclContexts = 0;

/// Number of TU Local decl contexts read/total
unsigned NumTULocalVisibleDeclContexts = 0,
TotalTULocalVisibleDeclContexts = 0;

/// Total size of modules, in bits, currently loaded
uint64_t TotalModulesSizeInBits = 0;

Expand Down Expand Up @@ -1481,6 +1496,9 @@ class ASTReader
const serialization::reader::ModuleLocalLookupTable *
getModuleLocalLookupTables(DeclContext *Primary) const;

const serialization::reader::DeclContextLookupTable *
getTULocalLookupTables(DeclContext *Primary) const;

/// Get the loaded specializations lookup tables for \p D,
/// if any.
serialization::reader::LazySpecializationInfoLookupTable *
Expand Down
11 changes: 9 additions & 2 deletions clang/include/clang/Serialization/ASTWriter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,9 @@ class ASTWriter : public ASTDeserializationListener,
/// file.
unsigned NumModuleLocalDeclContexts = 0;

/// The number of TULocal declcontexts written to the AST file.
unsigned NumTULocalDeclContexts = 0;

/// A mapping from each known submodule to its ID number, which will
/// be a positive integer.
llvm::DenseMap<const Module *, unsigned> SubmoduleIDs;
Expand Down Expand Up @@ -594,12 +597,14 @@ class ASTWriter : public ASTDeserializationListener,
void
GenerateNameLookupTable(ASTContext &Context, const DeclContext *DC,
llvm::SmallVectorImpl<char> &LookupTable,
llvm::SmallVectorImpl<char> &ModuleLocalLookupTable);
llvm::SmallVectorImpl<char> &ModuleLocalLookupTable,
llvm::SmallVectorImpl<char> &TULocalLookupTable);
uint64_t WriteDeclContextLexicalBlock(ASTContext &Context,
const DeclContext *DC);
void WriteDeclContextVisibleBlock(ASTContext &Context, DeclContext *DC,
uint64_t &VisibleBlockOffset,
uint64_t &ModuleLocalBlockOffset);
uint64_t &ModuleLocalBlockOffset,
uint64_t &TULocalBlockOffset);
void WriteTypeDeclOffsets();
void WriteFileDeclIDsMap();
void WriteComments(ASTContext &Context);
Expand Down Expand Up @@ -633,8 +638,10 @@ class ASTWriter : public ASTDeserializationListener,
unsigned DeclContextLexicalAbbrev = 0;
unsigned DeclContextVisibleLookupAbbrev = 0;
unsigned DeclModuleLocalVisibleLookupAbbrev = 0;
unsigned DeclTULocalLookupAbbrev = 0;
unsigned UpdateVisibleAbbrev = 0;
unsigned ModuleLocalUpdateVisibleAbbrev = 0;
unsigned TULocalUpdateVisibleAbbrev = 0;
unsigned DeclRecordAbbrev = 0;
unsigned DeclTypedefAbbrev = 0;
unsigned DeclVarAbbrev = 0;
Expand Down
89 changes: 74 additions & 15 deletions clang/lib/Serialization/ASTReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1425,10 +1425,9 @@ bool ASTReader::ReadLexicalDeclContextStorage(ModuleFile &M,
return false;
}

bool ASTReader::ReadVisibleDeclContextStorage(ModuleFile &M,
BitstreamCursor &Cursor,
uint64_t Offset, GlobalDeclID ID,
bool IsModuleLocal) {
bool ASTReader::ReadVisibleDeclContextStorage(
ModuleFile &M, BitstreamCursor &Cursor, uint64_t Offset, GlobalDeclID ID,
ASTReader::VisibleDeclContextStorageKind VisibleKind) {
assert(Offset != 0);

SavedStreamPosition SavedPosition(Cursor);
Expand All @@ -1452,22 +1451,42 @@ bool ASTReader::ReadVisibleDeclContextStorage(ModuleFile &M,
return true;
}
unsigned RecCode = MaybeRecCode.get();
if (!IsModuleLocal && RecCode != DECL_CONTEXT_VISIBLE) {
Error("Expected visible lookup table block");
return true;
}
if (IsModuleLocal && RecCode != DECL_CONTEXT_MODULE_LOCAL_VISIBLE) {
Error("Expected module local visible lookup table block");
return true;
switch (VisibleKind) {
case VisibleDeclContextStorageKind::GenerallyVisible:
if (RecCode != DECL_CONTEXT_VISIBLE) {
Error("Expected visible lookup table block");
return true;
}
break;
case VisibleDeclContextStorageKind::ModuleLocalVisible:
if (RecCode != DECL_CONTEXT_MODULE_LOCAL_VISIBLE) {
Error("Expected module local visible lookup table block");
return true;
}
break;
case VisibleDeclContextStorageKind::TULocalVisible:
if (RecCode != DECL_CONTEXT_TU_LOCAL_VISIBLE) {
Error("Expected TU local lookup table block");
return true;
}
break;
}

// We can't safely determine the primary context yet, so delay attaching the
// lookup table until we're done with recursive deserialization.
auto *Data = (const unsigned char*)Blob.data();
if (!IsModuleLocal)
switch (VisibleKind) {
case VisibleDeclContextStorageKind::GenerallyVisible:
PendingVisibleUpdates[ID].push_back(UpdateData{&M, Data});
else
break;
case VisibleDeclContextStorageKind::ModuleLocalVisible:
PendingModuleLocalVisibleUpdates[ID].push_back(UpdateData{&M, Data});
break;
case VisibleDeclContextStorageKind::TULocalVisible:
if (M.Kind == MK_MainFile)
TULocalUpdates[ID].push_back(UpdateData{&M, Data});
break;
}
return false;
}

Expand Down Expand Up @@ -3613,6 +3632,21 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
break;
}

case UPDATE_TU_LOCAL_VISIBLE: {
if (F.Kind != MK_MainFile)
break;
unsigned Idx = 0;
GlobalDeclID ID = ReadDeclID(F, Record, Idx);
auto *Data = (const unsigned char *)Blob.data();
TULocalUpdates[ID].push_back(UpdateData{&F, Data});
// If we've already loaded the decl, perform the updates when we finish
// loading this block.
if (Decl *D = GetExistingDecl(ID))
PendingUpdateRecords.push_back(
PendingUpdateRecord(ID, D, /*JustLoaded=*/false));
break;
}

case CXX_ADDED_TEMPLATE_SPECIALIZATION: {
unsigned Idx = 0;
GlobalDeclID ID = ReadDeclID(F, Record, Idx);
Expand Down Expand Up @@ -3717,6 +3751,7 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
TotalLexicalDeclContexts += Record[2];
TotalVisibleDeclContexts += Record[3];
TotalModuleLocalVisibleDeclContexts += Record[4];
TotalTULocalVisibleDeclContexts += Record[5];
break;

case UNUSED_FILESCOPED_DECLS:
Expand Down Expand Up @@ -4002,7 +4037,7 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
break;

case DELAYED_NAMESPACE_LEXICAL_VISIBLE_RECORD: {
if (Record.size() % 4 != 0)
if (Record.size() % 5 != 0)
return llvm::createStringError(
std::errc::illegal_byte_sequence,
"invalid DELAYED_NAMESPACE_LEXICAL_VISIBLE_RECORD block in AST "
Expand All @@ -4021,9 +4056,12 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
uint64_t LocalModuleLocalOffset = Record[I++];
uint64_t ModuleLocalOffset =
LocalModuleLocalOffset ? BaseOffset + LocalModuleLocalOffset : 0;
uint64_t TULocalLocalOffset = Record[I++];
uint64_t TULocalOffset =
TULocalLocalOffset ? BaseOffset + TULocalLocalOffset : 0;

DelayedNamespaceOffsetMap[ID] = {LexicalOffset, VisibleOffset,
ModuleLocalOffset};
ModuleLocalOffset, TULocalOffset};

assert(!GetExistingDecl(ID) &&
"We shouldn't load the namespace in the front of delayed "
Expand Down Expand Up @@ -8473,6 +8511,15 @@ bool ASTReader::FindExternalVisibleDeclsByName(const DeclContext *DC,
}
}

if (auto It = TULocalLookups.find(DC); It != TULocalLookups.end()) {
++NumTULocalVisibleDeclContexts;
for (GlobalDeclID ID : It->second.Table.find(Name)) {
NamedDecl *ND = cast<NamedDecl>(GetDecl(ID));
if (ND->getDeclName() == Name && Found.insert(ND).second)
Decls.push_back(ND);
}
}

SetExternalVisibleDeclsForName(DC, Name, Decls);
return !Decls.empty();
}
Expand Down Expand Up @@ -8500,6 +8547,7 @@ void ASTReader::completeVisibleDeclsMap(const DeclContext *DC) {

findAll(Lookups, NumVisibleDeclContextsRead);
findAll(ModuleLocalLookups, NumModuleLocalVisibleDeclContexts);
findAll(TULocalLookups, NumTULocalVisibleDeclContexts);

for (DeclsMap::iterator I = Decls.begin(), E = Decls.end(); I != E; ++I) {
SetExternalVisibleDeclsForName(DC, I->first, I->second);
Expand All @@ -8519,6 +8567,12 @@ ASTReader::getModuleLocalLookupTables(DeclContext *Primary) const {
return I == ModuleLocalLookups.end() ? nullptr : &I->second;
}

const serialization::reader::DeclContextLookupTable *
ASTReader::getTULocalLookupTables(DeclContext *Primary) const {
auto I = TULocalLookups.find(Primary);
return I == TULocalLookups.end() ? nullptr : &I->second;
}

serialization::reader::LazySpecializationInfoLookupTable *
ASTReader::getLoadedSpecializationsLookupTables(const Decl *D, bool IsPartial) {
assert(D->isCanonicalDecl());
Expand Down Expand Up @@ -8634,6 +8688,11 @@ void ASTReader::PrintStats() {
NumModuleLocalVisibleDeclContexts, TotalModuleLocalVisibleDeclContexts,
((float)NumModuleLocalVisibleDeclContexts /
TotalModuleLocalVisibleDeclContexts * 100));
if (TotalTULocalVisibleDeclContexts)
std::fprintf(stderr, " %u/%u visible declcontexts in GMF read (%f%%)\n",
NumTULocalVisibleDeclContexts, TotalTULocalVisibleDeclContexts,
((float)NumTULocalVisibleDeclContexts /
TotalTULocalVisibleDeclContexts * 100));
if (TotalNumMethodPoolEntries)
std::fprintf(stderr, " %u/%u method pool entries read (%f%%)\n",
NumMethodPoolEntriesRead, TotalNumMethodPoolEntries,
Expand Down
45 changes: 35 additions & 10 deletions clang/lib/Serialization/ASTReaderDecl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,8 @@ class ASTDeclReader : public DeclVisitor<ASTDeclReader, void> {
void VisitLifetimeExtendedTemporaryDecl(LifetimeExtendedTemporaryDecl *D);

void VisitDeclContext(DeclContext *DC, uint64_t &LexicalOffset,
uint64_t &VisibleOffset, uint64_t &ModuleLocalOffset);
uint64_t &VisibleOffset, uint64_t &ModuleLocalOffset,
uint64_t &TULocalOffset);

template <typename T>
RedeclarableResult VisitRedeclarable(Redeclarable<T> *D);
Expand Down Expand Up @@ -1859,7 +1860,9 @@ void ASTDeclReader::VisitHLSLBufferDecl(HLSLBufferDecl *D) {
uint64_t LexicalOffset = 0;
uint64_t VisibleOffset = 0;
uint64_t ModuleLocalOffset = 0;
VisitDeclContext(D, LexicalOffset, VisibleOffset, ModuleLocalOffset);
uint64_t TULocalOffset = 0;
VisitDeclContext(D, LexicalOffset, VisibleOffset, ModuleLocalOffset,
TULocalOffset);
D->IsCBuffer = Record.readBool();
D->KwLoc = readSourceLocation();
D->LBraceLoc = readSourceLocation();
Expand Down Expand Up @@ -2770,10 +2773,12 @@ void ASTDeclReader::VisitLifetimeExtendedTemporaryDecl(

void ASTDeclReader::VisitDeclContext(DeclContext *DC, uint64_t &LexicalOffset,
uint64_t &VisibleOffset,
uint64_t &ModuleLocalOffset) {
uint64_t &ModuleLocalOffset,
uint64_t &TULocalOffset) {
LexicalOffset = ReadLocalOffset();
VisibleOffset = ReadLocalOffset();
ModuleLocalOffset = ReadLocalOffset();
TULocalOffset = ReadLocalOffset();
}

template <typename T>
Expand Down Expand Up @@ -3903,6 +3908,7 @@ Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) {
case DECL_CONTEXT_LEXICAL:
case DECL_CONTEXT_VISIBLE:
case DECL_CONTEXT_MODULE_LOCAL_VISIBLE:
case DECL_CONTEXT_TU_LOCAL_VISIBLE:
case DECL_SPECIALIZATIONS:
case DECL_PARTIAL_SPECIALIZATIONS:
llvm_unreachable("Record cannot be de-serialized with readDeclRecord");
Expand Down Expand Up @@ -4213,9 +4219,10 @@ Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) {
uint64_t LexicalOffset = 0;
uint64_t VisibleOffset = 0;
uint64_t ModuleLocalOffset = 0;
uint64_t TULocalOffset = 0;

Reader.VisitDeclContext(DC, LexicalOffset, VisibleOffset,
ModuleLocalOffset);
Reader.VisitDeclContext(DC, LexicalOffset, VisibleOffset, ModuleLocalOffset,
TULocalOffset);

// Get the lexical and visible block for the delayed namespace.
// It is sufficient to judge if ID is in DelayedNamespaceOffsetMap.
Expand All @@ -4227,18 +4234,24 @@ Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) {
LexicalOffset = Iter->second.LexicalOffset;
VisibleOffset = Iter->second.VisibleOffset;
ModuleLocalOffset = Iter->second.ModuleLocalOffset;
TULocalOffset = Iter->second.TULocalOffset;
}

if (LexicalOffset &&
ReadLexicalDeclContextStorage(*Loc.F, DeclsCursor, LexicalOffset, DC))
return nullptr;
if (VisibleOffset &&
ReadVisibleDeclContextStorage(*Loc.F, DeclsCursor, VisibleOffset, ID,
/*IsModuleLocal=*/false))
if (VisibleOffset && ReadVisibleDeclContextStorage(
*Loc.F, DeclsCursor, VisibleOffset, ID,
VisibleDeclContextStorageKind::GenerallyVisible))
return nullptr;
if (ModuleLocalOffset &&
ReadVisibleDeclContextStorage(*Loc.F, DeclsCursor, ModuleLocalOffset,
ID, /*IsModuleLocal=*/true))
ReadVisibleDeclContextStorage(
*Loc.F, DeclsCursor, ModuleLocalOffset, ID,
VisibleDeclContextStorageKind::ModuleLocalVisible))
return nullptr;
if (TULocalOffset && ReadVisibleDeclContextStorage(
*Loc.F, DeclsCursor, TULocalOffset, ID,
VisibleDeclContextStorageKind::TULocalVisible))
return nullptr;
}
assert(Record.getIdx() == Record.size());
Expand Down Expand Up @@ -4404,6 +4417,18 @@ void ASTReader::loadDeclUpdateRecords(PendingUpdateRecord &Record) {
DC->setHasExternalVisibleStorage(true);
}

if (auto I = TULocalUpdates.find(ID); I != TULocalUpdates.end()) {
auto Updates = std::move(I->second);
TULocalUpdates.erase(I);

auto *DC = cast<DeclContext>(D)->getPrimaryContext();
for (const auto &Update : Updates)
TULocalLookups[DC].Table.add(
Update.Mod, Update.Data,
reader::ASTDeclContextNameLookupTrait(*this, *Update.Mod));
DC->setHasExternalVisibleStorage(true);
}

// Load any pending related decls.
if (D->isCanonicalDecl()) {
if (auto IT = RelatedDeclsMap.find(ID); IT != RelatedDeclsMap.end()) {
Expand Down
Loading

0 comments on commit fb2c9d9

Please sign in to comment.