Skip to content

Commit b5ef3bd

Browse files
tamasvajktamasvajk
authored
Merge pull request github#4516 from tamasvajk/feature/attributes
C#: Refactor attribute extraction
2 parents 86fc9e6 + 453c97a commit b5ef3bd

38 files changed

+840
-204
lines changed

‎csharp/change-notes/2020-10-21-rework-attribute-extraction.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
lgtm,codescanning
2+
* Attribute extraction has been extended to extract attributes not only from source
3+
code, but from referenced assemblies too. This change may lead to more results in
4+
queries that rely on attributes. Note that, as more attributes might be extracted,
5+
the DB size might increase.

‎csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -198,12 +198,12 @@ public void AnalyseTree(SyntaxTree tree)
198198
/// Perform an analysis on an assembly.
199199
/// </summary>
200200
/// <param name="assembly">Assembly to analyse.</param>
201-
private void AnalyseAssembly(PortableExecutableReference assembly)
201+
private void AnalyseReferenceAssembly(PortableExecutableReference assembly)
202202
{
203203
// CIL first - it takes longer.
204204
if (options.CIL)
205205
extractionTasks.Add(() => DoExtractCIL(assembly));
206-
extractionTasks.Add(() => DoAnalyseAssembly(assembly));
206+
extractionTasks.Add(() => DoAnalyseReferenceAssembly(assembly));
207207
}
208208

209209
private static bool FileIsUpToDate(string src, string dest)
@@ -250,7 +250,7 @@ private void DoAnalyseCompilation(string cwd, string[] args)
250250
/// extraction within the snapshot.
251251
/// </summary>
252252
/// <param name="r">The assembly to extract.</param>
253-
private void DoAnalyseAssembly(PortableExecutableReference r)
253+
private void DoAnalyseReferenceAssembly(PortableExecutableReference r)
254254
{
255255
try
256256
{
@@ -294,6 +294,8 @@ private void DoAnalyseAssembly(PortableExecutableReference r)
294294
AnalyseNamespace(cx, module.GlobalNamespace);
295295
}
296296

297+
Entities.Attribute.ExtractAttributes(cx, assembly, Extraction.Entities.Assembly.Create(cx, assembly.GetSymbolLocation()));
298+
297299
cx.PopulateAll();
298300
}
299301
}
@@ -335,7 +337,7 @@ public void AnalyseReferences()
335337
{
336338
foreach (var r in compilation.References.OfType<PortableExecutableReference>())
337339
{
338-
AnalyseAssembly(r);
340+
AnalyseReferenceAssembly(r);
339341
}
340342
}
341343

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs

Lines changed: 91 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,82 +1,134 @@
1+
using System.IO;
2+
using System.Linq;
13
using Microsoft.CodeAnalysis;
24
using Microsoft.CodeAnalysis.CSharp.Syntax;
35
using Semmle.Extraction.Entities;
4-
using System.IO;
56

67
namespace Semmle.Extraction.CSharp.Entities
78
{
8-
internal class Attribute : FreshEntity, IExpressionParentEntity
9+
internal class Attribute : CachedEntity<AttributeData>, IExpressionParentEntity
910
{
1011
bool IExpressionParentEntity.IsTopLevelParent => true;
1112

12-
private readonly AttributeData attribute;
13+
private readonly AttributeSyntax attributeSyntax;
1314
private readonly IEntity entity;
1415

15-
public Attribute(Context cx, AttributeData attribute, IEntity entity)
16-
: base(cx)
16+
private Attribute(Context cx, AttributeData attributeData, IEntity entity)
17+
: base(cx, attributeData)
1718
{
18-
this.attribute = attribute;
19+
this.attributeSyntax = attributeData.ApplicationSyntaxReference?.GetSyntax() as AttributeSyntax;
1920
this.entity = entity;
20-
TryPopulate();
2121
}
2222

23-
protected override void Populate(TextWriter trapFile)
23+
public override void WriteId(TextWriter trapFile)
2424
{
25-
if (attribute.ApplicationSyntaxReference != null)
25+
if (ReportingLocation?.IsInSource == true)
26+
{
27+
trapFile.WriteSubId(Location);
28+
trapFile.Write(";attribute");
29+
}
30+
else
2631
{
27-
// !! Extract attributes from assemblies.
28-
// This is harder because the "expression" entities presume the
29-
// existence of a syntax tree. This is not the case for compiled
30-
// attributes.
31-
var syntax = attribute.ApplicationSyntaxReference.GetSyntax() as AttributeSyntax;
32-
ExtractAttribute(cx.TrapWriter.Writer, syntax, attribute.AttributeClass, entity);
32+
trapFile.Write('*');
3333
}
3434
}
3535

36-
public Attribute(Context cx, AttributeSyntax attribute, IEntity entity)
37-
: base(cx)
36+
public override void WriteQuotedId(TextWriter trapFile)
3837
{
39-
var info = cx.GetSymbolInfo(attribute);
40-
ExtractAttribute(cx.TrapWriter.Writer, attribute, info.Symbol.ContainingType, entity);
38+
if (ReportingLocation?.IsInSource == true)
39+
{
40+
base.WriteQuotedId(trapFile);
41+
}
42+
else
43+
{
44+
trapFile.Write('*');
45+
}
4146
}
4247

43-
private void ExtractAttribute(System.IO.TextWriter trapFile, AttributeSyntax syntax, ITypeSymbol attributeClass, IEntity entity)
48+
public override void Populate(TextWriter trapFile)
4449
{
45-
var type = Type.Create(cx, attributeClass);
50+
var type = Type.Create(Context, symbol.AttributeClass);
4651
trapFile.attributes(this, type.TypeRef, entity);
52+
trapFile.attribute_location(this, Location);
4753

48-
trapFile.attribute_location(this, cx.Create(syntax.Name.GetLocation()));
54+
if (attributeSyntax is object)
55+
{
56+
if (Context.Extractor.OutputPath != null)
57+
{
58+
trapFile.attribute_location(this, Assembly.CreateOutputAssembly(Context));
59+
}
60+
61+
TypeMention.Create(Context, attributeSyntax.Name, this, type);
62+
}
4963

50-
if (cx.Extractor.OutputPath != null)
51-
trapFile.attribute_location(this, Assembly.CreateOutputAssembly(cx));
64+
ExtractArguments(trapFile);
65+
}
5266

53-
TypeMention.Create(cx, syntax.Name, this, type);
67+
private void ExtractArguments(TextWriter trapFile)
68+
{
69+
var childIndex = 0;
70+
foreach (var constructorArgument in symbol.ConstructorArguments)
71+
{
72+
CreateExpressionFromArgument(
73+
constructorArgument,
74+
attributeSyntax?.ArgumentList.Arguments[childIndex].Expression,
75+
this,
76+
childIndex++);
77+
}
5478

55-
if (syntax.ArgumentList != null)
79+
foreach (var namedArgument in symbol.NamedArguments)
5680
{
57-
cx.PopulateLater(() =>
81+
var expr = CreateExpressionFromArgument(
82+
namedArgument.Value,
83+
attributeSyntax?.ArgumentList.Arguments.Single(a => a.NameEquals?.Name?.Identifier.Text == namedArgument.Key).Expression,
84+
this,
85+
childIndex++);
86+
87+
if (expr is object)
5888
{
59-
var child = 0;
60-
foreach (var arg in syntax.ArgumentList.Arguments)
61-
{
62-
var expr = Expression.Create(cx, arg.Expression, this, child++);
63-
if (!(arg.NameEquals is null))
64-
{
65-
trapFile.expr_argument_name(expr, arg.NameEquals.Name.Identifier.Text);
66-
}
67-
}
68-
});
89+
trapFile.expr_argument_name(expr, namedArgument.Key);
90+
}
6991
}
7092
}
7193

94+
private Expression CreateExpressionFromArgument(TypedConstant constant, ExpressionSyntax syntax, IExpressionParentEntity parent,
95+
int childIndex)
96+
{
97+
return syntax is null
98+
? Expression.CreateGenerated(Context, constant, parent, childIndex, Location)
99+
: Expression.Create(Context, syntax, parent, childIndex);
100+
}
101+
102+
public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel;
103+
104+
public override Microsoft.CodeAnalysis.Location ReportingLocation => attributeSyntax?.Name.GetLocation();
105+
106+
private Semmle.Extraction.Entities.Location ___location;
107+
private Semmle.Extraction.Entities.Location Location =>
108+
___location ?? (___location = Semmle.Extraction.Entities.Location.Create(Context, attributeSyntax is null ? entity.ReportingLocation : attributeSyntax.Name.GetLocation()));
109+
110+
public override bool NeedsPopulation => true;
111+
72112
public static void ExtractAttributes(Context cx, ISymbol symbol, IEntity entity)
73113
{
74114
foreach (var attribute in symbol.GetAttributes())
75115
{
76-
new Attribute(cx, attribute, entity);
116+
Create(cx, attribute, entity);
77117
}
78118
}
79119

80-
public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel;
120+
public static Attribute Create(Context cx, AttributeData attributeData, IEntity entity)
121+
{
122+
var init = (attributeData, entity);
123+
return AttributeFactory.Instance.CreateEntity(cx, attributeData, init);
124+
}
125+
126+
private class AttributeFactory : ICachedEntityFactory<(AttributeData attributeData, IEntity receiver), Attribute>
127+
{
128+
public static readonly AttributeFactory Instance = new AttributeFactory();
129+
130+
public Attribute Create(Context cx, (AttributeData attributeData, IEntity receiver) init) =>
131+
new Attribute(cx, init.attributeData, init.receiver);
132+
}
81133
}
82134
}

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Expression.cs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
using Microsoft.CodeAnalysis;
22
using Microsoft.CodeAnalysis.CSharp;
33
using Microsoft.CodeAnalysis.CSharp.Syntax;
4+
using Semmle.Extraction.CSharp.Entities.Expressions;
45
using Semmle.Extraction.CSharp.Populators;
56
using Semmle.Extraction.Entities;
67
using Semmle.Extraction.Kinds;
8+
using System;
79
using System.IO;
810
using System.Linq;
911

@@ -125,6 +127,42 @@ public static void CreateDeferred(Context cx, ExpressionSyntax node, IExpression
125127
private static bool ContainsPattern(SyntaxNode node) =>
126128
node is PatternSyntax || node is VariableDesignationSyntax || node.ChildNodes().Any(ContainsPattern);
127129

130+
/// <summary>
131+
/// Creates a generated expression from a typed constant.
132+
/// </summary>
133+
public static Expression CreateGenerated(Context cx, TypedConstant constant, IExpressionParentEntity parent,
134+
int childIndex, Semmle.Extraction.Entities.Location ___location)
135+
{
136+
if (constant.IsNull)
137+
{
138+
return Literal.CreateGeneratedNullLiteral(cx, parent, childIndex, ___location);
139+
}
140+
141+
switch (constant.Kind)
142+
{
143+
case TypedConstantKind.Primitive:
144+
return Literal.CreateGenerated(cx, parent, childIndex, constant.Type, constant.Value, ___location);
145+
case TypedConstantKind.Enum:
146+
// Enum value is generated in the following format: (Enum)value
147+
Action<Expression, int> createChild = (parent, index) => Literal.CreateGenerated(cx, parent, index, ((INamedTypeSymbol)constant.Type).EnumUnderlyingType, constant.Value, ___location);
148+
var cast = Cast.CreateGenerated(cx, parent, childIndex, constant.Type, constant.Value, createChild, ___location);
149+
return cast;
150+
case TypedConstantKind.Type:
151+
var type = ((ITypeSymbol)constant.Value).OriginalDefinition;
152+
return TypeOf.CreateGenerated(cx, parent, childIndex, type, ___location);
153+
case TypedConstantKind.Array:
154+
// Single dimensional arrays are in the following format:
155+
// * new Type[N] { item1, item2, ..., itemN }
156+
// * new Type[0]
157+
//
158+
// itemI is generated recursively.
159+
return NormalArrayCreation.CreateGenerated(cx, parent, childIndex, constant.Type, constant.Values, ___location);
160+
default:
161+
cx.ExtractionError("Couldn't extract constant in attribute", constant.ToString(), ___location);
162+
return null;
163+
}
164+
}
165+
128166
/// <summary>
129167
/// Adapt the operator kind depending on whether it's a dynamic call or a user-operator call.
130168
/// </summary>

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArrayCreation.cs

Lines changed: 38 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using Microsoft.CodeAnalysis.CSharp;
33
using Microsoft.CodeAnalysis.CSharp.Syntax;
44
using Semmle.Extraction.Kinds;
5+
using System.Collections.Generic;
56
using System.IO;
67
using System.Linq;
78

@@ -10,6 +11,8 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
1011
internal abstract class ArrayCreation<TSyntaxNode> : Expression<TSyntaxNode>
1112
where TSyntaxNode : ExpressionSyntax
1213
{
14+
protected const int InitializerIndex = -1;
15+
1316
protected ArrayCreation(ExpressionNodeInfo info) : base(info) { }
1417
}
1518

@@ -48,7 +51,7 @@ protected override void PopulateExpression(TextWriter trapFile)
4851

4952
if (!(Initializer is null))
5053
{
51-
ArrayInitializer.Create(new ExpressionNodeInfo(cx, Initializer, this, -1));
54+
ArrayInitializer.Create(new ExpressionNodeInfo(cx, Initializer, this, InitializerIndex));
5255
}
5356

5457
if (explicitlySized)
@@ -66,17 +69,7 @@ private void SetArraySizes(InitializerExpressionSyntax initializer, int rank)
6669
return;
6770
}
6871

69-
var info = new ExpressionInfo(
70-
cx,
71-
new AnnotatedType(Entities.Type.Create(cx, cx.Compilation.GetSpecialType(Microsoft.CodeAnalysis.SpecialType.System_Int32)), NullableAnnotation.None),
72-
Location,
73-
ExprKind.INT_LITERAL,
74-
this,
75-
level,
76-
true,
77-
initializer.Expressions.Count.ToString());
78-
79-
new Expression(info);
72+
Literal.CreateGenerated(cx, this, level, cx.Compilation.GetSpecialType(SpecialType.System_Int32), initializer.Expressions.Count, Location);
8073

8174
initializer = initializer.Expressions.FirstOrDefault() as InitializerExpressionSyntax;
8275
}
@@ -92,6 +85,37 @@ private NormalArrayCreation(ExpressionNodeInfo info) : base(info) { }
9285
public override InitializerExpressionSyntax Initializer => Syntax.Initializer;
9386

9487
public static Expression Create(ExpressionNodeInfo info) => new NormalArrayCreation(info).TryPopulate();
88+
89+
public static Expression CreateGenerated(Context cx, IExpressionParentEntity parent, int childIndex, ITypeSymbol type, IEnumerable<TypedConstant> items, Semmle.Extraction.Entities.Location ___location)
90+
{
91+
var info = new ExpressionInfo(
92+
cx,
93+
new AnnotatedType(Entities.Type.Create(cx, type), NullableAnnotation.None),
94+
___location,
95+
ExprKind.ARRAY_CREATION,
96+
parent,
97+
childIndex,
98+
true,
99+
null);
100+
101+
var arrayCreation = new Expression(info);
102+
103+
var length = items.Count();
104+
105+
Literal.CreateGenerated(cx, arrayCreation, 0, cx.Compilation.GetSpecialType(SpecialType.System_Int32), length, ___location);
106+
107+
if (length > 0)
108+
{
109+
var arrayInit = ArrayInitializer.CreateGenerated(cx, arrayCreation, InitializerIndex, ___location);
110+
var child = 0;
111+
foreach (var item in items)
112+
{
113+
Expression.CreateGenerated(cx, item, arrayInit, child++, ___location);
114+
}
115+
}
116+
117+
return arrayCreation;
118+
}
95119
}
96120

97121
internal class StackAllocArrayCreation : ExplicitArrayCreation<StackAllocArrayCreationExpressionSyntax>
@@ -119,7 +143,7 @@ private ImplicitStackAllocArrayCreation(ExpressionNodeInfo info) : base(info.Set
119143

120144
protected override void PopulateExpression(TextWriter trapFile)
121145
{
122-
ArrayInitializer.Create(new ExpressionNodeInfo(cx, Syntax.Initializer, this, -1));
146+
ArrayInitializer.Create(new ExpressionNodeInfo(cx, Syntax.Initializer, this, InitializerIndex));
123147
trapFile.implicitly_typed_array_creation(this);
124148
trapFile.stackalloc_array_creation(this);
125149
}
@@ -135,7 +159,7 @@ protected override void PopulateExpression(TextWriter trapFile)
135159
{
136160
if (Syntax.Initializer != null)
137161
{
138-
ArrayInitializer.Create(new ExpressionNodeInfo(cx, Syntax.Initializer, this, -1));
162+
ArrayInitializer.Create(new ExpressionNodeInfo(cx, Syntax.Initializer, this, InitializerIndex));
139163
}
140164

141165
trapFile.implicitly_typed_array_creation(this);

0 commit comments

Comments
 (0)