Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
b600255
C#: Enable overlay compilation in lib/qlpack.yml.
michaelnebel Oct 21, 2025
bcc6234
C#: Fix bad join due to overlay compilation.
michaelnebel Oct 29, 2025
b48ce8d
C#: Add overlay builtins to the dbscheme.
michaelnebel Sep 12, 2025
cfb6843
C#: Add upgrade- and downgrade scripts.
michaelnebel Sep 12, 2025
9026a5a
C#: Turn on overlay support in codeql-extractor.yml.
michaelnebel Sep 12, 2025
aa80558
C#: Add functionality to detect overlay mode and integrate in extract…
michaelnebel Sep 12, 2025
cab9d81
C#: Add unit test.
michaelnebel Sep 12, 2025
61f6512
C#: Sprinkle uses of OnlyScaffold to extract less when in overlay mode.
michaelnebel Sep 17, 2025
76ac2df
C#: Write overlay metadata at end of extraction.
michaelnebel Oct 21, 2025
121c150
C#: Define discarding predicates for expressions, statements, locatio…
michaelnebel Oct 23, 2025
8a34421
C#: Add QL overlay tests.
michaelnebel Oct 27, 2025
40035d2
C#: Add a locatable type to the dbscheme.
michaelnebel Oct 27, 2025
8927b0a
C#: Simplify the discarding and also discard type mentions and comments.
michaelnebel Oct 27, 2025
04d33cb
C#: Add discarding for diagnostics and extractor messages.
michaelnebel Oct 28, 2025
1d2f154
C#: Add change-note.
michaelnebel Oct 30, 2025
504bb9c
C#: Only scaffold assemblies in overlay mode, only extract expression…
michaelnebel Nov 3, 2025
d95ebc7
C#: Add using directives and type mentions as star entities.
michaelnebel Nov 3, 2025
7c670cd
C#: Address review comments and make more early returns in Populate.
michaelnebel Nov 7, 2025
1657dfb
C#: Remove expression population safeguard and guard creation of cons…
michaelnebel Nov 7, 2025
9d300e3
C#: Address comments in the QL implementation.
michaelnebel Nov 7, 2025
ded1328
C#: Do not extract comments when scaffolding.
michaelnebel Nov 10, 2025
43118ec
C#: The extraction of the TypeMentions for return type and explicit i…
michaelnebel Nov 10, 2025
0a16cf6
C#: Do not require that comments and type locations are in source in …
michaelnebel Nov 10, 2025
c44b747
C#: Minor code quality improvements.
michaelnebel Nov 10, 2025
d6b7424
C#: Add the same strategy in as in Java for XML element discarding.
michaelnebel Nov 10, 2025
3492811
C#: Add XML overlay tests.
michaelnebel Nov 10, 2025
0541dcc
C#: Add discarding for ASP elements.
michaelnebel Nov 10, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@ private CommentBlock(Context cx, Comments.CommentBlock init)
public override void Populate(TextWriter trapFile)
{
trapFile.commentblock(this);
if (!Context.OnlyScaffold)
Symbol.CommentLines.ForEach((l, child) => trapFile.commentblock_child(this, l, child));
if (Context.OnlyScaffold)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would have thought that we could simply move this early return up to the beginning of Populate?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually, now that I think about it, shouldn't we instead be guarding all the places where we extract comments (such as calls to CommentPopulator.ExtractCommentBlocks,CommentPopulator.ExtractComment, and inside Context.BindComments)?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would like to move this to the begging on the populate method and then also guard the BindTo method. This should be internally consistent and not cause any dangling references (I will also push the scaffolding guard from CachedEntitiy to the Context for comment binding).
Will add one more commit for this.

Also note that for TypeMentions, the guard has been moved to the top of the populate method (as creating these will not cause any dangling references). Most places in the populate methods the TypeMention creation has been "moved below" the scaffolding early return check (and thus not be created), but I haven't done so in all places as this will create a large amount of extra guarding.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, I will also put in the guards the places where you suggest to avoid all the string processing before the actual comments are created.

{
WriteLocationToTrap(trapFile.commentblock_location, this, Context.CreateLocation(Symbol.Location));
return;
}
Symbol.CommentLines.ForEach((l, child) => trapFile.commentblock_child(this, l, child));
WriteLocationToTrap(trapFile.commentblock_location, this, Context.CreateLocation(Symbol.Location));
}

public override bool NeedsPopulation => true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,14 @@ private CommentLine(Context cx, Microsoft.CodeAnalysis.Location loc, CommentLine

public override void Populate(TextWriter trapFile)
{
location = Context.CreateLocation(Location);
trapFile.commentline(this, Type == CommentLineType.MultilineContinuation ? CommentLineType.Multiline : Type, Text, RawText);
if (!Context.OnlyScaffold)
if (Context.OnlyScaffold)
{
WriteLocationToTrap(trapFile.commentline_location, this, location);
return;
}
location = Context.CreateLocation(Location);
WriteLocationToTrap(trapFile.commentline_location, this, location);

}

public override Microsoft.CodeAnalysis.Location? ReportingLocation => location?.Symbol;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ public CompilerDiagnostic(Context cx, Microsoft.CodeAnalysis.Diagnostic diag, Co

protected override void Populate(TextWriter trapFile)
{
if (Context.OnlyScaffold)
{
return;
}

var key = diagnostic.Id;
var messageCount = compilation.messageCounts.AddOrUpdate(key, 1, (_, c) => c + 1);
if (messageCount > limit)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,17 @@ public override void Populate(TextWriter trapFile)
ContainingType!.PopulateGenerics();

trapFile.constructors(this, Symbol.ContainingType.Name, ContainingType, (Constructor)OriginalDefinition);
if (Context.ExtractLocation(Symbol) && (!IsDefault || IsBestSourceLocation))

if (Symbol.IsImplicitlyDeclared)
{
WriteLocationToTrap(trapFile.constructor_location, this, Location);
var lineCounts = new LineCounts() { Total = 2, Code = 1, Comment = 0 };
trapFile.numlines(this, lineCounts);
}
ExtractCompilerGenerated(trapFile);

if (Context.OnlyScaffold)
{
return;
}

if (MakeSynthetic)
Expand All @@ -40,12 +48,11 @@ public override void Populate(TextWriter trapFile)
Statements.SyntheticEmptyBlock.Create(Context, this, 0, Location);
}

if (Symbol.IsImplicitlyDeclared)
if (Context.ExtractLocation(Symbol) && (!IsDefault || IsBestSourceLocation))
{
var lineCounts = new LineCounts() { Total = 2, Code = 1, Comment = 0 };
trapFile.numlines(this, lineCounts);
WriteLocationToTrap(trapFile.constructor_location, this, Location);
}
ExtractCompilerGenerated(trapFile);

}

protected override void ExtractInitializers(TextWriter trapFile)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ public override void Populate(TextWriter trapFile)
ContainingType!.PopulateGenerics();

trapFile.destructors(this, $"~{Symbol.ContainingType.Name}", ContainingType, OriginalDefinition(Context, this, Symbol));

if (Context.ExtractLocation(Symbol))
{
WriteLocationToTrap(trapFile.destructor_location, this, Location);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ public override void Populate(TextWriter trapFile)
Method.Create(Context, remover);

PopulateModifiers(trapFile);
BindComments();

var declSyntaxReferences = IsSourceDeclaration
? Symbol.DeclaringSyntaxReferences.Select(d => d.GetSyntax()).ToArray()
Expand All @@ -51,6 +50,13 @@ public override void Populate(TextWriter trapFile)
TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier!.Name, this, explicitInterface);
}

if (Context.OnlyScaffold)
{
return;
}

BindComments();

if (Context.ExtractLocation(Symbol))
{
WriteLocationsToTrap(trapFile.event_location, this, Locations);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ private ExtractionMessage(Context cx, Message msg, bool bypassLimit) : base(cx)

protected override void Populate(TextWriter trapFile)
{
if (Context.OnlyScaffold)
{
return;
}

// For the time being we're counting the number of messages per severity, we could introduce other groupings in the future
var key = msg.Severity.ToString();
groupedMessageCounts.AddOrUpdate(key, 1, (_, c) => c + 1);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,17 @@ public override void Populate(TextWriter trapFile)
}
}

if (Context.OnlyScaffold)
{
return;
}

if (Context.ExtractLocation(Symbol))
{
WriteLocationsToTrap(trapFile.field_location, this, Locations);
}

if (!IsSourceDeclaration || !Symbol.FromSource() || Context.OnlyScaffold)
if (!IsSourceDeclaration || !Symbol.FromSource())
return;

Context.BindComments(this, Location.Symbol);
Expand Down
44 changes: 22 additions & 22 deletions csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,6 @@ public override void Populate(TextWriter trapFile)

var type = Type.Create(Context, Symbol.Type);
trapFile.indexers(this, Symbol.GetName(useMetadataName: true), ContainingType!, type.TypeRef, OriginalDefinition);
if (Context.ExtractLocation(Symbol))
{
WriteLocationsToTrap(trapFile.indexer_location, this, Locations);
}

var getter = BodyDeclaringSymbol.GetMethod;
var setter = BodyDeclaringSymbol.SetMethod;
Expand All @@ -42,27 +38,9 @@ public override void Populate(TextWriter trapFile)
Parameter.Create(Context, Symbol.Parameters[i], this, original);
}

if (IsSourceDeclaration && !Context.OnlyScaffold)
{
var expressionBody = ExpressionBody;
if (expressionBody is not null)
{
// The expression may need to reference parameters in the getter.
// So we need to arrange that the expression is populated after the getter.
Context.PopulateLater(() => Expression.CreateFromNode(new ExpressionNodeInfo(Context, expressionBody, this, 0).SetType(Symbol.GetAnnotatedType())));
}
}

PopulateAttributes();
PopulateModifiers(trapFile);

if (Context.OnlyScaffold)
{
return;
}

BindComments();

var declSyntaxReferences = IsSourceDeclaration
? Symbol.DeclaringSyntaxReferences.
Select(d => d.GetSyntax()).OfType<IndexerDeclarationSyntax>().ToArray()
Expand All @@ -76,6 +54,28 @@ public override void Populate(TextWriter trapFile)
TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier!.Name, this, explicitInterface);
}

if (Context.OnlyScaffold)
{
return;
}

if (Context.ExtractLocation(Symbol))
{
WriteLocationsToTrap(trapFile.indexer_location, this, Locations);
}

if (IsSourceDeclaration)
{
var expressionBody = ExpressionBody;
if (expressionBody is not null)
{
// The expression may need to reference parameters in the getter.
// So we need to arrange that the expression is populated after the getter.
Context.PopulateLater(() => Expression.CreateFromNode(new ExpressionNodeInfo(Context, expressionBody, this, 0).SetType(Symbol.GetAnnotatedType())));
}
}

BindComments();

foreach (var syntax in declSyntaxReferences)
TypeMention.Create(Context, syntax.Type, this, type);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ public void PopulateManual(Expression parent, bool isVar)
trapFile.localvars(this, Kinds.VariableKind.None, Symbol.Name, @var, Type.Create(Context, parent.Type).TypeRef, parent);
}

if (Context.OnlyScaffold)
{
return;
}

WriteLocationToTrap(trapFile.localvar_location, this, Location);

DefineConstantValue(trapFile);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ public void Overrides(TextWriter trapFile)
{
trapFile.explicitly_implements(this, explicitInterface.TypeRef);

if (IsSourceDeclaration && !Context.OnlyScaffold)
if (IsSourceDeclaration)
{
foreach (var syntax in Symbol.DeclaringSyntaxReferences.Select(d => d.GetSyntax()).OfType<MethodDeclarationSyntax>())
TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier!.Name, this, explicitInterface);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ public override void Populate(TextWriter trapFile)

var ns = Namespace.Create(Context, @namespace);
trapFile.namespace_declarations(this, ns);
WriteLocationToTrap(trapFile.namespace_declaration_location, this, Context.CreateLocation(node.Name.GetLocation()));

var visitor = new Populators.TypeOrNamespaceVisitor(Context, trapFile, this);

Expand All @@ -48,6 +47,12 @@ public override void Populate(TextWriter trapFile)
{
trapFile.parent_namespace_declaration(this, parent);
}

if (Context.OnlyScaffold)
{
return;
}
WriteLocationToTrap(trapFile.namespace_declaration_location, this, Context.CreateLocation(node.Name.GetLocation()));
}

public static NamespaceDeclaration Create(Context cx, BaseNamespaceDeclarationSyntax decl, NamespaceDeclaration parent)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,17 @@ public override void Populate(TextWriter trapFile)
var returnType = Type.Create(Context, Symbol.ReturnType);
trapFile.methods(this, Name, ContainingType, returnType.TypeRef, OriginalDefinition);

if (IsSourceDeclaration && !Context.OnlyScaffold)
PopulateGenerics(trapFile);
Overrides(trapFile);
ExtractRefReturn(trapFile, Symbol, this);
ExtractCompilerGenerated(trapFile);

if (Context.OnlyScaffold)
{
return;
}

if (IsSourceDeclaration)
{
foreach (var declaration in Symbol.DeclaringSyntaxReferences.Select(s => s.GetSyntax()).OfType<MethodDeclarationSyntax>())
{
Expand All @@ -47,11 +57,6 @@ public override void Populate(TextWriter trapFile)
{
WriteLocationsToTrap(trapFile.method_location, this, Locations);
}

PopulateGenerics(trapFile);
Overrides(trapFile);
ExtractRefReturn(trapFile, Symbol, this);
ExtractCompilerGenerated(trapFile);
}

private bool IsCompilerGeneratedDelegate() =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,11 @@ public override void Populate(TextWriter trapFile)
var type = Type.Create(Context, Symbol.Type);
trapFile.@params(this, Name, type.TypeRef, Ordinal, ParamKind, Parent!, Original);

if (Context.OnlyScaffold)
{
return;
}

if (Context.ExtractLocation(Symbol))
{
var locations = Context.GetLocations(Symbol);
Expand All @@ -140,7 +145,7 @@ Symbol.ContainingSymbol is IMethodSymbol ms &&
Context.PopulateLater(defaultValueExpressionCreation);
}

if (!IsSourceDeclaration || !Symbol.FromSource() || Context.OnlyScaffold)
if (!IsSourceDeclaration || !Symbol.FromSource())
return;

BindComments();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,15 @@ public sealed override void Populate(TextWriter trapFile)
PopulatePreprocessor(trapFile);

trapFile.preprocessor_directive_active(this, Symbol.IsActive);
WriteLocationToTrap(trapFile.preprocessor_directive_location, this, Context.CreateLocation(ReportingLocation));

var compilation = Compilation.Create(Context);
trapFile.preprocessor_directive_compilation(this, compilation);

if (Context.OnlyScaffold)
{
return;
}
WriteLocationToTrap(trapFile.preprocessor_directive_location, this, Context.CreateLocation(ReportingLocation));
}

protected abstract void PopulatePreprocessor(TextWriter trapFile);
Expand Down
10 changes: 8 additions & 2 deletions csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ public override void Populate(TextWriter trapFile)
{
PopulateAttributes();
PopulateModifiers(trapFile);
BindComments();
PopulateNullability(trapFile, Symbol.GetAnnotatedType());
PopulateRefKind(trapFile, Symbol.RefKind);

Expand Down Expand Up @@ -69,12 +68,19 @@ public override void Populate(TextWriter trapFile)
TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier!.Name, this, explicitInterface);
}

if (Context.OnlyScaffold)
{
return;
}

BindComments();

if (Context.ExtractLocation(Symbol))
{
WriteLocationsToTrap(trapFile.property_location, this, Locations);
}

if (IsSourceDeclaration && Symbol.FromSource() && !Context.OnlyScaffold)
if (IsSourceDeclaration && Symbol.FromSource())
{
var expressionBody = ExpressionBody;
if (expressionBody is not null)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ private static Type GetArrayElementType(Type type)

protected override void Populate(TextWriter trapFile)
{
if (Context.OnlyScaffold)
{
return;
}

switch (syntax.Kind())
{
case SyntaxKind.ArrayType:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,14 @@ private DynamicType(Context cx, IDynamicTypeSymbol init)
public override void Populate(TextWriter trapFile)
{
trapFile.types(this, Kinds.TypeKind.DYNAMIC, "dynamic");
WriteLocationToTrap(trapFile.type_location, this, Location);

trapFile.has_modifiers(this, Modifier.Create(Context, "public"));
trapFile.parent_namespace(this, Namespace.Create(Context, Context.Compilation.GlobalNamespace));
if (Context.OnlyScaffold)
{
return;
}
WriteLocationToTrap(trapFile.type_location, this, Location);
}

public override void WriteId(EscapingTextWriter trapFile)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ public override void Populate(TextWriter trapFile)
trapFile.tuple_element(this, index++, element);
}

if (Context.OnlyScaffold)
{
return;
}
// Note: symbol.Locations seems to be very inconsistent
// about what locations are available for a tuple type.
// Sometimes it's the source code, and sometimes it's empty.
Expand Down
Loading