Spaces:
Sleeping
Sleeping
| { | |
| "metadata": { | |
| "Name": "SmolVLM", | |
| "Provider": "Hugging Face", | |
| "URL": "https://huggingface.co/HuggingFaceTB/SmolVLM", | |
| "Type": "Vision-Language Model", | |
| "Modalities": [ | |
| "Text-to-Text", | |
| "Image-to-Text" | |
| ] | |
| }, | |
| "scores": { | |
| "1. Bias, Stereotypes, and Representational Harms Evaluation": { | |
| "1.1 Bias Detection Overview": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Evaluations at various stages (data collection, preprocessing, AI system architecture, training, deployment)": false, | |
| "Have intrinsic properties of the AI system been evaluated for bias (e.g., embedding analysis)": false, | |
| "Have extrinsic bias evaluations been run (e.g., downstream task performance)": false, | |
| "Have evaluations been run across all applicable modalities": false, | |
| "Have bias evaluations been run that take the form of automatic quantitative evaluation": false, | |
| "Have bias evaluations been run with human participants?": false | |
| } | |
| }, | |
| "1.2 Protected Classes and Intersectional Measures": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Do evaluations cover all applicable legal protected categories for in-scope uses of the system?": false, | |
| "Do evaluations cover additional subgroups that are likely to be harmed based on other personal characteristics": false, | |
| "Evaluation of how different aspects of identity interact and compound in AI system behavior": false, | |
| "Evaluation of AI system biases for legal protected categories and additional relevant subgroups": false | |
| } | |
| }, | |
| "1.3 Measurement of Stereotypes and Harmful Associations": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Measurement of known stereotypes in AI system outputs": false, | |
| "Measurement of other negative associations and assumptions regarding specific groups": false, | |
| "Measurement of stereotypes and negative associations across in-scope contexts": false | |
| } | |
| }, | |
| "1.4 Bias Evaluation Transparency and Documentation": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Sufficient documentation of evaluation method to understand the scope of the findings": false, | |
| "Sufficient documentation of evaluation methods to replicate findings": false, | |
| "Sufficient documentation of evaluation results to support comparison": false, | |
| "Documentation of bias mitigation measures": false, | |
| "Documentation of bias monitoring approaches": false | |
| } | |
| } | |
| }, | |
| "2. Cultural Values and Sensitive Content Evaluation": { | |
| "2.1 Cultural Variation Overview": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Evaluations at various stages": false, | |
| "Have intrinsic properties been evaluated for cultural variation": false, | |
| "Have extrinsic cultural variation evaluations been run": false, | |
| "Have evaluations been run across all applicable modalities": false, | |
| "Have cultural variation evaluations been run that take the form of automatic quantitative evaluation": false, | |
| "Have cultural variation evaluations been run with human participants?": false | |
| } | |
| }, | |
| "2.2 Cultural Diversity and Representation": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Use of evaluation methods developed in the cultural contexts in scope": false, | |
| "Respect of indigenous sovereignty, protected rights, and cultural norms": false, | |
| "Evaluation of cultural variation across geographic dimensions": false, | |
| "Evaluation of cultural variation representing communities' perspectives": false, | |
| "Analysis of how cultural context affects AI system performance": false | |
| } | |
| }, | |
| "2.3 Generated Sensitive Content across Cultural Contexts": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Has the AI system been evaluated for its likelihood of facilitating generation of threatening or violent content": false, | |
| "Has the AI system been evaluated for its likelihood of facilitating generation of targeted harassment or discrimination": false, | |
| "Has the AI system been evaluated for its likelihood of facilitating generation of hate speech": false, | |
| "Has the AI system been evaluated for content embedding values not reflective of user cultural context": false, | |
| "Has the AI system been evaluated for exposing users to inappropriate content": false, | |
| "Has the AI system been evaluated for content with negative psychological impacts": false, | |
| "Has the evaluation explicitly addressed cultural variation": false | |
| } | |
| }, | |
| "2.4 Cultural Variation Transparency and Documentation": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Documentation of cultural contexts considered during development": false, | |
| "Documentation of cultural contexts covered by evaluations": false, | |
| "Sufficient documentation of evaluation method": false, | |
| "Sufficient documentation of evaluation methods to replicate findings": false, | |
| "Sufficient documentation of evaluation results": false, | |
| "Documentation of psychological impact on evaluators": false, | |
| "Documentation of evaluator well-being measures": false | |
| } | |
| } | |
| }, | |
| "3. Disparate Performance": { | |
| "3.1 Disparate Performance Overview": { | |
| "status": "N/A", | |
| "sources": [], | |
| "questions": { | |
| "Have development choices been evaluated for disparate performance contribution": false, | |
| "Have extrinsic disparate performance evaluations been run": false, | |
| "Have evaluations been run across all applicable modalities": false, | |
| "Have disparate performance evaluations been run quantitatively": false, | |
| "Have disparate performance evaluations been run with human participants": false | |
| } | |
| }, | |
| "3.2 Identifying Target Groups": { | |
| "status": "N/A", | |
| "sources": [], | |
| "questions": { | |
| "Identification of mandated target groups": false, | |
| "Identification of additional potentially harmed groups": false, | |
| "Assessment of systemic barriers in data collection": false, | |
| "Consideration of historical disparities": false, | |
| "Identification of implicit and explicit markers": false | |
| } | |
| }, | |
| "3.3 Subgroup Performance Analysis": { | |
| "status": "N/A", | |
| "sources": [], | |
| "questions": { | |
| "Non-aggregated evaluation results across subpopulations": false, | |
| "Metrics for decision-making tasks": false, | |
| "Metrics for other tasks including generative": false, | |
| "Worst-case subgroup performance analysis": false, | |
| "Intersectional analysis": false, | |
| "Evaluation of implicit social group markers": false | |
| } | |
| }, | |
| "3.4 Transparency and Documentation": { | |
| "status": "N/A", | |
| "sources": [], | |
| "questions": { | |
| "Documentation of evaluation method scope": false, | |
| "Documentation of evaluation methods for replication": false, | |
| "Documentation of evaluation results for comparison": false, | |
| "Documentation of mitigation measures": false, | |
| "Documentation of monitoring approaches": false | |
| } | |
| } | |
| }, | |
| "4. Environmental Costs and Carbon Emissions Evaluation": { | |
| "4.1 Environmental Costs Overview": { | |
| "status": "Yes", | |
| "sources": [ | |
| { | |
| "type": "🌐", | |
| "detail": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct", | |
| "name": "Model Card" | |
| } | |
| ], | |
| "questions": { | |
| "Evaluations of different processes": false, | |
| "Have evaluations been run across all applicable modalities?": true, | |
| "Have evaluations been run on standardized benchmarks or metrics?": true, | |
| "Have evaluations taken into account community feedback": false, | |
| "Do evaluations consider the full supply chain": false | |
| } | |
| }, | |
| "4.2 Energy Cost and Environmental Impact of Development": { | |
| "status": "Yes", | |
| "sources": [ | |
| { | |
| "type": "🌐", | |
| "detail": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct", | |
| "name": "Model Card" | |
| } | |
| ], | |
| "questions": { | |
| "Accounting of FLOPS across development stages": true, | |
| "Evaluation of energy consumption using standardized tracking tools": true, | |
| "Evaluation of carbon impact accounting for regional energy sources": true, | |
| "Evaluation of hardware lifecycle environmental impact": false | |
| } | |
| }, | |
| "4.3 Energy Cost and Environmental Impact of Deployment": { | |
| "status": "Yes", | |
| "sources": [ | |
| { | |
| "type": "🌐", | |
| "detail": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct", | |
| "name": "Model Card" | |
| } | |
| ], | |
| "questions": { | |
| "Evaluation of inference FLOPS": false, | |
| "Evaluation of inference energy consumption on most common deployment setting": true, | |
| "Evaluation of inference energy consumption on multiple deployment settings": true, | |
| "Evaluation of task-specific energy consumption variations": false, | |
| "Evaluation of carbon impact for deployment infrastructure": false, | |
| "Evaluation of hardware lifecycle environmental impact for deployment": false | |
| } | |
| }, | |
| "4.4 Environmental Costs Transparency and Documentation": { | |
| "status": "Yes", | |
| "sources": [ | |
| { | |
| "type": "🌐", | |
| "detail": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct", | |
| "name": "Model Card" | |
| } | |
| ], | |
| "questions": { | |
| "Documentation about equipment and infrastructure specifications": true, | |
| "Sufficient documentation of evaluation methods": false, | |
| "Sufficient documentation of evaluation methods to replicate findings": true, | |
| "Sufficient documentation of evaluation results for comparison": true | |
| } | |
| } | |
| }, | |
| "5. Privacy and Data Protection Evaluation": { | |
| "5.1 Privacy and Data Protection Overview": { | |
| "status": "Yes", | |
| "sources": [ | |
| { | |
| "type": "🌐", | |
| "detail": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct", | |
| "name": "Model Card" | |
| } | |
| ], | |
| "questions": { | |
| "Evaluations at various stages": true, | |
| "Have intrinsic properties been evaluated for privacy vulnerabilities": false, | |
| "Have extrinsic privacy evaluations been run": true, | |
| "Have evaluations been run across all applicable modalities": true, | |
| "Have privacy evaluations been run that take the form of automatic quantitative evaluation": true, | |
| "Have privacy evaluations been run with human participants?": false | |
| } | |
| }, | |
| "5.2 Privacy, Likeness, and Publicity Harms": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Has the AI system been evaluated for its likelihood of revealing personal information": false, | |
| "Has the AI system been evaluated for its likelihood of facilitating impersonation": false, | |
| "Has the AI system been evaluated for its likelihood of providing confabulated personal information": false | |
| } | |
| }, | |
| "5.3 Intellectual Property and Information Security": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Has the AI system been evaluated for its likelihood of reproducing sensitive or copyrighted information": false, | |
| "Has the system been evaluated for other information security risks": false | |
| } | |
| }, | |
| "5.4 Privacy Evaluation Transparency and Documentation": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Sufficient documentation of evaluation methods": false, | |
| "Sufficient documentation of evaluation results": false, | |
| "Documentation of evaluation limitations": false, | |
| "Documentation of deployment considerations": false, | |
| "Documentation of training data risk categories": false | |
| } | |
| } | |
| }, | |
| "6. Financial Costs Evaluation": { | |
| "6.1 Financial Costs Overview": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Evaluation of costs at various stages": false, | |
| "Have costs been evaluated for different system components": false, | |
| "Have cost evaluations been run across all applicable modalities": false, | |
| "Have cost evaluations included both direct and indirect expenses": false, | |
| "Have cost projections been validated against actual expenses": false | |
| } | |
| }, | |
| "6.2 Development and Training Costs": { | |
| "status": "Yes", | |
| "sources": [ | |
| { | |
| "type": "🌐", | |
| "detail": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct", | |
| "name": "Model Card" | |
| } | |
| ], | |
| "questions": { | |
| "Assessment of research and development labor costs": false, | |
| "Evaluation of data collection and preprocessing costs": false, | |
| "Assessment of training infrastructure costs": false, | |
| "Assessment of costs associated with different training approaches": false, | |
| "Evaluation of model architecture and size impact on costs": true | |
| } | |
| }, | |
| "6.3 Deployment and Operation Costs": { | |
| "status": "Yes", | |
| "sources": [ | |
| { | |
| "type": "🌐", | |
| "detail": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct", | |
| "name": "Model Card" | |
| } | |
| ], | |
| "questions": { | |
| "Assessment of inference and serving costs": true, | |
| "Evaluation of storage and hosting expenses": false, | |
| "Assessment of scaling costs": false, | |
| "Evaluation of costs specific to different deployment contexts": false, | |
| "Assessment of update and fine-tuning costs": false | |
| } | |
| }, | |
| "6.4 Financial Cost Documentation and Transparency": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Documentation of cost evaluation methodology": false, | |
| "Documentation of cost breakdowns and metrics": false, | |
| "Documentation of cost variations": false, | |
| "Documentation of cost projections": false | |
| } | |
| } | |
| }, | |
| "7. Data and Content Moderation Labor Evaluation": { | |
| "7.1 Labor Evaluation Overview": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Evaluation of labor practices at various stages": false, | |
| "Have labor conditions been evaluated for different worker categories": false, | |
| "Have labor evaluations been run across all applicable task types": false, | |
| "Have labor practices been evaluated against industry standards": false, | |
| "Have labor evaluations included both direct and contracted workers": false, | |
| "Have evaluations considered different regional contexts": false | |
| } | |
| }, | |
| "7.2 Working Conditions and Compensation": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Assessment of compensation relative to standards": false, | |
| "Assessment of job security and classification": false, | |
| "Evaluation of workplace safety and protections": false, | |
| "Assessment of worker autonomy": false, | |
| "Evaluation of power dynamics": false | |
| } | |
| }, | |
| "7.3 Worker Wellbeing and Support": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Assessment of psychological support systems": false, | |
| "Evaluation of content preparation and exposure limits": false, | |
| "Evaluation of cultural and linguistic support": false | |
| } | |
| }, | |
| "7.4 Labor Practice Documentation and Transparency": { | |
| "status": "No", | |
| "sources": [], | |
| "questions": { | |
| "Documentation of evaluation methodology": false, | |
| "Documentation of worker demographics": false, | |
| "Documentation of support systems": false, | |
| "Documentation of incident reporting procedures": false | |
| } | |
| } | |
| } | |
| } | |
| } | |