Ollama benchmark Q2 2024 - Exoscale A40 - Consumption
Go back to listAs power consumption is one of the main aspect of performance, we obtained several measurements of usage through the gpu-burn software:
- GFLOPS: As therorical raw performance value
- Temperature: Average hotness during the operation
- Watt: Average energy delivery
{
"configuration": {
"chart": {
"type": "spline",
"polar": false,
"zoomType": "",
"options3d": {},
"height": null,
"width": null,
"margin": null,
"inverted": false,
"zooming": {}
},
"credits": {
"enabled": false
},
"title": {
"text": ""
},
"colorAxis": null,
"subtitle": {
"text": ""
},
"xAxis": {
"title": {
"text": [
"Memory usage"
],
"useHTML": false,
"style": {
"color": "#666666"
}
},
"categories": [
10,
20,
30,
40,
50,
60,
70,
80,
90
],
"lineWidth": 1,
"tickInterval": null,
"tickWidth": 1,
"tickLength": 10,
"tickPixelInterval": null,
"plotLines": null,
"labels": {
"format": "{value}%"
},
"plotBands": null,
"visible": true,
"floor": null,
"ceiling": null,
"type": "linear",
"min": null,
"gridLineWidth": null,
"gridLineColor": null,
"minorTickInterval": null,
"minorTickWidth": 0,
"minTickInterval": null,
"startOnTick": true,
"endOnTick": null,
"tickmarkPlacement": null,
"units": null,
"minRange": null,
"crosshair": true
},
"yAxis": [
{
"title": {
"text": "Power usage",
"style": {
"color": "#3c8c40"
}
},
"categories": [],
"plotLines": null,
"plotBands": null,
"lineWidth": null,
"tickInterval": 100,
"tickLength": 10,
"floor": null,
"ceiling": null,
"gridLineInterpolation": null,
"gridLineWidth": 1,
"gridLineColor": "#CCC",
"min": 0,
"max": null,
"minorTickInterval": null,
"minorTickWidth": 0,
"minTickInterval": null,
"startOnTick": true,
"endOnTick": null,
"minRange": null,
"type": "linear",
"tickmarkPlacement": null,
"labels": {
"format": "{value} W",
"style": {
"color": "#3c8c40"
}
}
},
{
"title": {
"text": "Temperature",
"style": {
"color": "#3c0c40"
}
},
"categories": [],
"plotLines": null,
"plotBands": null,
"lineWidth": null,
"tickInterval": 100,
"tickLength": 10,
"floor": null,
"ceiling": null,
"gridLineInterpolation": null,
"gridLineWidth": 1,
"gridLineColor": "#CCC",
"min": 0,
"max": null,
"minorTickInterval": null,
"minorTickWidth": 0,
"minTickInterval": null,
"startOnTick": true,
"endOnTick": null,
"minRange": null,
"type": "linear",
"tickmarkPlacement": null,
"labels": {
"format": "{value} C\u00b0",
"style": {
"color": "#3c0c40"
}
},
"opposite": true
},
{
"title": {
"text": "GFLOPS",
"style": {
"color": "#Ac8c40"
}
},
"categories": [],
"plotLines": null,
"plotBands": null,
"lineWidth": null,
"tickInterval": 100,
"tickLength": 10,
"floor": null,
"ceiling": null,
"gridLineInterpolation": null,
"gridLineWidth": 0,
"gridLineColor": "#CCC",
"min": 0,
"max": null,
"minorTickInterval": null,
"minorTickWidth": 0,
"minTickInterval": null,
"startOnTick": true,
"endOnTick": null,
"minRange": null,
"type": "linear",
"tickmarkPlacement": null,
"labels": {
"format": "{value}",
"style": {
"color": "#Ac8c40"
}
}
}
],
"zAxis": {
"title": {
"text": ""
}
},
"plotOptions": {
"series": {
"dataLabels": {
"enabled": false,
"format": null,
"distance": 30,
"align": "center",
"inside": null,
"allowOverlap": false,
"style": {
"fontSize": "11px"
}
},
"showInLegend": null,
"turboThreshold": 1000,
"stacking": "",
"groupPadding": 0,
"centerInCategory": false,
"findNearestPointBy": "x"
}
},
"navigator": {
"enabled": false
},
"scrollbar": {
"enabled": false
},
"rangeSelector": {
"enabled": false,
"inputEnabled": false
},
"legend": {
"enabled": true,
"maxHeight": null,
"align": "center",
"verticalAlign": "bottom",
"layout": "horizontal",
"width": null,
"margin": 12,
"reversed": false
},
"series": [
{
"name": "Power usage",
"yAxis": 0,
"color": "#3c8c40",
"data": [
142.17857142857142,
220.55357142857142,
220.58928571428572,
207.25,
219.82142857142856,
217.02604166666666,
230.27777777777777,
237.66666666666669,
243.28095238095239
],
"lineWidth": 5,
"tooltip": {
"valueSuffix": " W"
}
},
{
"name": "Temperature",
"yAxis": 1,
"color": "#3c0c40",
"data": [
57.37662337662337,
61.05194805194805,
63.467532467532465,
67.27272727272727,
70.24675324675326,
71.39448051948052,
72.64583333333333,
73.4040404040404,
74.2611111111111
],
"lineWidth": 5,
"tooltip": {
"valueSuffix": " C\u00b0"
}
},
{
"name": "GFLOPS",
"yAxis": 2,
"color": "#Ac8c40",
"data": [
21891.103896103894,
22301.896103896106,
22053.87012987013,
22200.080808080806,
22253.376623376622,
22008.22023809524,
22111.34375,
22015.242424242428,
22112.409523809525
],
"lineWidth": 5,
"tooltip": {
"valueSuffix": " GFLOPS"
}
}
],
"drilldown": {},
"tooltip": {
"enabled": true,
"useHTML": false,
"format": null,
"headerFormat": "",
"pointFormat": "<span style=\"color:{series.color}\">{series.name}</span>: <b>{point.y:.2f}</b><br/>",
"footerFormat": "",
"shared": true,
"outside": false,
"valueDecimals": null,
"split": false
},
"annotations": null
},
"hc_type": "chart",
"id": "212034850675425857494640883846537400314"
}
Notes
- Please keep in mind that GFLOPS doesn't directly reflect what LLM can directly provides
- Power consumption is not in the original sofware. Cloud Mercato did a pull request to include this value in results. Feel free to help us to push this contribution
- An NVIDIA A40 has idle temperature of 33 C° and 14 Watts
- No errors were produced by GPU computations during our tests