Skip to content

Commit

Permalink
edited reports to validate schema
Browse files Browse the repository at this point in the history
  • Loading branch information
@shubhobm
shubhobm committed Mar 13, 2023
1 parent 2b16df4 commit bef099f
Show file tree
Hide file tree
Showing 26 changed files with 222 additions and 170 deletions.
16 changes: 5 additions & 11 deletions reports/2022/AVID-2022-R0001.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,15 @@
"deployer": [
"HuggingFace"
],
"artifact": [
"artifacts": [
{
"type": "Model",
"name": "bert-base-uncased"
}
]
},
"problemtype": {
"class": "LLM evaluation",
"classof": "LLM Evaluation",
"type": "Detection",
"description": {
"lang": "eng",
Expand All @@ -32,7 +32,7 @@
"sensitive": "gender"
},
"detection_method": {
"class": "Significance test",
"type": "Significance Test",
"name": "Two-sample z test for proportions"
}
}
Expand Down Expand Up @@ -65,16 +65,10 @@
"Ethics"
],
"sep_view": [
{
"id": "E0101",
"name": "Group Fairness"
}
"E0101: Group fairness"
],
"lifecycle_view": [
{
"id": "L05",
"stage": "Evaluation"
}
"L05: Evaluation"
],
"taxonomy_version": "0.1"
}
Expand Down
16 changes: 5 additions & 11 deletions reports/2022/AVID-2022-R0002.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,15 @@
"deployer": [
"HuggingFace"
],
"artifact": [
"artifacts": [
{
"type": "Model",
"name": "xlm-roberta-base"
}
]
},
"problemtype": {
"class": "LLM evaluation",
"classof": "LLM Evaluation",
"type": "Detection",
"description": {
"lang": "eng",
Expand All @@ -32,7 +32,7 @@
"sensitive": "gender"
},
"detection_method": {
"class": "Significance test",
"type": "Significance Test",
"name": "Two-sample z test for proportions"
}
}
Expand Down Expand Up @@ -65,16 +65,10 @@
"Ethics"
],
"sep_view": [
{
"id": "E0101",
"name": "Group Fairness"
}
"E0101: Group fairness"
],
"lifecycle_view": [
{
"id": "L05",
"stage": "Evaluation"
}
"L05: Evaluation"
],
"taxonomy_version": "0.1"
}
Expand Down
18 changes: 6 additions & 12 deletions reports/2022/AVID-2022-R0003.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"deployer": [
"HuggingFace"
],
"artifact": [
"artifacts": [
{
"type": "Model",
"name": "bert-base-uncased"
Expand All @@ -25,7 +25,7 @@
]
},
"problemtype": {
"class": "LLM evaluation",
"classof": "LLM Evaluation",
"type": "Detection",
"description": {
"lang": "eng",
Expand All @@ -39,8 +39,8 @@
"measured": "Probability of predicting biased pronouns",
"sensitive": "gender"
},
"detection": {
"class": "Significance test",
"detection_method": {
"type": "Significance Test",
"name": "One-sample z test for mean"
}
}
Expand Down Expand Up @@ -73,16 +73,10 @@
"Ethics"
],
"sep_view": [
{
"id": "E0101",
"name": "Group Fairness"
}
"E0101: Group fairness"
],
"lifecycle_view": [
{
"id": "L05",
"stage": "Evaluation"
}
"L05: Evaluation"
],
"taxonomy_version": "0.1"
}
Expand Down
18 changes: 6 additions & 12 deletions reports/2022/AVID-2022-R0004.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"deployer": [
"HuggingFace"
],
"artifact": [
"artifacts": [
{
"type": "Model",
"name": "xlm-roberta-base"
Expand All @@ -25,7 +25,7 @@
]
},
"problemtype": {
"class": "LLM evaluation",
"classof": "LLM Evaluation",
"type": "Detection",
"description": {
"lang": "eng",
Expand All @@ -39,8 +39,8 @@
"measured": "Probability of predicting biased pronouns",
"sensitive": "gender"
},
"detection": {
"class": "Significance test",
"detection_method": {
"type": "Significance Test",
"name": "One-sample z test for mean"
}
}
Expand Down Expand Up @@ -73,16 +73,10 @@
"Ethics"
],
"sep_view": [
{
"id": "E0101",
"name": "Group Fairness"
}
"E0101: Group fairness"
],
"lifecycle_view": [
{
"id": "L05",
"stage": "Evaluation"
}
"L05: Evaluation"
],
"taxonomy_version": "0.1"
}
Expand Down
18 changes: 6 additions & 12 deletions reports/2022/AVID-2022-R0005.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@
"deployer": [
"HuggingFace"
],
"artifact": [
"artifacts": [
{
"type": "Model",
"name": "EleutherAI/gpt-neo-125M"
}
]
},
"problemtype": {
"class": "LLM evaluation",
"classof": "LLM Evaluation",
"type": "Detection",
"description": {
"lang": "eng",
Expand All @@ -33,8 +33,8 @@
"measured": "Toxicity score",
"sensitive": "religious_ideology"
},
"detection": {
"class": "Significance test",
"detection_method": {
"type": "Significance Test",
"name": "Anderson-Darling k-sample test"
}
}
Expand Down Expand Up @@ -67,16 +67,10 @@
"Ethics"
],
"sep_view": [
{
"id": "E0101",
"name": "Group Fairness"
}
"E0101: Group fairness"
],
"lifecycle_view": [
{
"id": "L05",
"stage": "Evaluation"
}
"L05: Evaluation"
],
"taxonomy_version": "0.1"
}
Expand Down
10 changes: 6 additions & 4 deletions reports/review/AML.CS0000.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@
"deployer": [
"Palo Alto Networks malware detection system"
],
"artifacts": {
"type": "system",
"name": "Palo Alto Networks malware detection system"
}
"artifacts": [
{
"type": "System",
"name": "Palo Alto Networks malware detection system"
}
]
},
"problemtype": {
"classof": "ATLAS Case Study",
Expand Down
10 changes: 6 additions & 4 deletions reports/review/AML.CS0001.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@
"deployer": [
"Palo Alto Networks ML-based DGA detection module"
],
"artifacts": {
"type": "system",
"name": "Palo Alto Networks ML-based DGA detection module"
}
"artifacts": [
{
"type": "System",
"name": "Palo Alto Networks ML-based DGA detection module"
}
]
},
"problemtype": {
"classof": "ATLAS Case Study",
Expand Down
20 changes: 12 additions & 8 deletions reports/review/AML.CS0002.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@
"deployer": [
"VirusTotal"
],
"artifacts": {
"type": "system",
"name": "VirusTotal"
}
"artifacts": [
{
"type": "System",
"name": "VirusTotal"
}
]
},
"problemtype": {
"classof": "ATLAS Case Study",
Expand All @@ -33,9 +35,11 @@
"value": "McAfee Advanced Threat Research noticed an increase in reports of a certain ransomware family that was out of the ordinary. Case investigation revealed that many samples of that particular ransomware family were submitted through a popular virus-sharing platform within a short amount of time. Further investigation revealed that based on string similarity the samples were all equivalent, and based on code similarity they were between 98 and 74 percent similar. Interestingly enough, the compile time was the same for all the samples. After more digging, researchers discovered that someone used 'metame' a metamorphic code manipulating tool to manipulate the original file towards mutant variants. The variants would not always be executable, but are still classified as the same ransomware family."
},
"impact": null,
"credit": {
"lang": "eng",
"value": "McAfee Advanced Threat Research"
},
"credit": [
{
"lang": "eng",
"value": "McAfee Advanced Threat Research"
}
],
"reported_date": "2020-01-01"
}
10 changes: 6 additions & 4 deletions reports/review/AML.CS0003.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@
"deployer": [
"CylancePROTECT, Cylance Smart Antivirus"
],
"artifacts": {
"type": "system",
"name": "CylancePROTECT, Cylance Smart Antivirus"
}
"artifacts": [
{
"type": "System",
"name": "CylancePROTECT, Cylance Smart Antivirus"
}
]
},
"problemtype": {
"classof": "ATLAS Case Study",
Expand Down
20 changes: 12 additions & 8 deletions reports/review/AML.CS0004.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@
"deployer": [
"Shanghai government tax office's facial recognition service"
],
"artifacts": {
"type": "system",
"name": "Shanghai government tax office's facial recognition service"
}
"artifacts": [
{
"type": "System",
"name": "Shanghai government tax office's facial recognition service"
}
]
},
"problemtype": {
"classof": "ATLAS Case Study",
Expand Down Expand Up @@ -38,9 +40,11 @@
"value": "This type of camera hijack attack can evade the traditional live facial recognition authentication model and enable access to privileged systems and victim impersonation.\n\nTwo individuals in China used this attack to gain access to the local government's tax system. They created a fake shell company and sent invoices via tax system to supposed clients. The individuals started this scheme in 2018 and were able to fraudulently collect $77 million.\n"
},
"impact": null,
"credit": {
"lang": "eng",
"value": "Ant Group AISEC Team"
},
"credit": [
{
"lang": "eng",
"value": "Ant Group AISEC Team"
}
],
"reported_date": "2020-01-01"
}
10 changes: 6 additions & 4 deletions reports/review/AML.CS0005.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@
"deployer": [
"Google Translate, Bing Translator, Systran Translate"
],
"artifacts": {
"type": "system",
"name": "Google Translate, Bing Translator, Systran Translate"
}
"artifacts": [
{
"type": "System",
"name": "Google Translate, Bing Translator, Systran Translate"
}
]
},
"problemtype": {
"classof": "ATLAS Case Study",
Expand Down
10 changes: 6 additions & 4 deletions reports/review/AML.CS0006.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@
"deployer": [
"Clearview AI facial recognition tool"
],
"artifacts": {
"type": "system",
"name": "Clearview AI facial recognition tool"
}
"artifacts": [
{
"type": "System",
"name": "Clearview AI facial recognition tool"
}
]
},
"problemtype": {
"classof": "ATLAS Case Study",
Expand Down
Loading

0 comments on commit bef099f

Please sign in to comment.