Merge pull request #735 from mlco2/codecarbon_v3_rc

Codecarbon version 3.0.0

Author: benoit-cty

.github/workflows/deploy.yml

@@ -44,4 +44,4 @@ jobs:
         APP_NAME: cc_dashboard_prod
       run: |
         ./clever-tools-latest_linux/clever link $CLEVER_APP_ID
-        ./clever-tools-latest_linux/clever deploy -f -a $APP_NAME --quiet
+        ./clever-tools-latest_linux/clever deploy -f -a $APP_NAME --quiet

.github/workflows/package.yml

@@ -93,4 +93,4 @@ jobs:
               run: mamba install --channel file:///tmp/conda-bld --channel codecarbon codecarbon
             - name: Test conda package
               shell: bash -l {0}
-              run: codecarbon --help
+              run: codecarbon --help

.github/workflows/release-drafter.yml

@@ -3,14 +3,13 @@ name: Release Drafter
 on:
   push:
     # branches to consider in the event; optional, defaults to all
-    branches:
-      - master
+    branches: [master]
 
 jobs:
   update_release_draft:
     runs-on: ubuntu-latest
     steps:
       # Drafts your next Release notes as Pull Requests are merged into "master"
-      - uses: release-drafter/release-drafter@v5.7.0
+      - uses: release-drafter/release-drafter@v6
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CONTRIBUTING.md

@@ -250,9 +250,10 @@ Here is the launch.json to be able to debug examples and tests:
 {
     "version": "0.2.0",
     "configurations": [
+
         {
             "name": "Python: Current File",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "program": "${file}",
             "console": "integratedTerminal",
@@ -261,13 +262,17 @@ Here is the launch.json to be able to debug examples and tests:
         },
         {
             "name": "PyTest: Current File",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "module": "pytest",
-            "args": ["${file}"],
+            "args": [
+                "-s",
+                "${file}"
+            ],
             "console": "integratedTerminal",
             "justMyCode": true,
-            "env": { "PYTHONPATH": "${workspaceRoot}" }
+            "env": { "PYTHONPATH": "${workspaceRoot}",
+            "CODECARBON_ALLOW_MULTIPLE_RUNS": "True"  }
         },
         {
             "name": "PyTest: codecarbon monitor",
@@ -279,7 +284,7 @@ Here is the launch.json to be able to debug examples and tests:
             ],
             "console": "integratedTerminal",
             "justMyCode": true,
-            "env": { "PYTHONPATH": "${workspaceRoot}" }
+            "env": { "PYTHONPATH": "${workspaceRoot}"}
         }
     ]
 }
@@ -350,7 +355,7 @@ to regenerate the html files.
 ### Release process
 
 -   Merge all PRs.
--   Create a PR bumping the version with `hatch run dev:bumpver update --patch`.
+-   Create a PR bumping the version with `hatch run dev:bumpver update --patch`. For a release candidate, use `hatch run dev:bumpver update --set-version  3.0.0_rc1`.
 -   Run `hatch run python3 .github/check_version.py` to check version consistancy.
 -   Update the dependencies with `hatch-pip-compile --upgrade --all`.
 -   [Build Documentation](#documentation) if needed with `hatch run docs:build`.

README.md

@@ -172,3 +172,9 @@ Here is a sample for BibTeX:
 # Contact 📝
 
 Maintainers are [@vict0rsch](https://github.com/vict0rsch) [@benoit-cty](https://github.com/benoit-cty) and [@SaboniAmine](https://github.com/saboniamine). Codecarbon is developed by volunteers from [**Mila**](http://mila.quebec) and the [**DataForGoodFR**](https://twitter.com/dataforgood_fr) community alongside donated professional time of engineers at [**Comet.ml**](https://comet.ml) and [**BCG GAMMA**](https://www.bcg.com/en-nl/beyond-consulting/bcg-gamma/default).
+
+## Star History
+
+Comparison of the number of stars accumulated by the different Python CO2 emissions projects:
+[![Star History Chart](https://api.star-history.com/svg?repos=mlco2/codecarbon,lfwa/carbontracker,sb-ai-lab/Eco2AI,fvaleye/tracarbon,Breakend/experiment-impact-tracker&type=Date)](https://star-history.com/#mlco2/codecarbon&lfwa/carbontracker&sb-ai-lab/Eco2AI&fvaleye/tracarbon&Breakend/experiment-impact-tracker&Date)
+

codecarbon/_version.py

@@ -1 +1 @@
-__version__ = "2.8.4"
+__version__ = "3.0.0"

codecarbon/core/api_client.py

@@ -264,8 +264,8 @@ def _create_run(self, experiment_id: str):
                 gpu_count=self.conf.get("gpu_count"),
                 gpu_model=self.conf.get("gpu_model"),
                 # Reduce precision for Privacy
-                longitude=round(self.conf.get("longitude"), 1),
-                latitude=round(self.conf.get("latitude"), 1),
+                longitude=round(self.conf.get("longitude", 0), 1),
+                latitude=round(self.conf.get("latitude", 0), 1),
                 region=self.conf.get("region"),
                 provider=self.conf.get("provider"),
                 ram_total_size=self.conf.get("ram_total_size"),

codecarbon/core/cpu.py

@@ -5,12 +5,14 @@
 """
 
 import os
+import re
 import shutil
 import subprocess
 import sys
 from typing import Dict, Optional, Tuple
 
 import pandas as pd
+import psutil
 from rapidfuzz import fuzz, process, utils
 
 from codecarbon.core.rapl import RAPLFile
@@ -19,6 +21,9 @@
 from codecarbon.external.logger import logger
 from codecarbon.input import DataSource
 
+# default W value per core for a CPU if no model is found in the ref csv
+DEFAULT_POWER_PER_CORE = 4
+
 
 def is_powergadget_available() -> bool:
     """
@@ -58,6 +63,24 @@ def is_rapl_available() -> bool:
         return False
 
 
+def is_psutil_available():
+    try:
+        nice = psutil.cpu_times().nice
+        if nice > 0.0001:
+            return True
+        else:
+            logger.debug(
+                f"is_psutil_available() : psutil.cpu_times().nice is too small : {nice} !"
+            )
+            return False
+    except Exception as e:
+        logger.debug(
+            "Not using the psutil interface, an exception occurred while instantiating "
+            + f"psutil.cpu_times : {e}",
+        )
+        return False
+
+
 class IntelPowerGadget:
     """
     A class to interface with Intel Power Gadget for monitoring CPU power consumption on Windows and (non-Apple Silicon) macOS.
@@ -237,7 +260,7 @@ class IntelRAPL:
 
     """
 
-    def __init__(self, rapl_dir="/sys/class/powercap/intel-rapl"):
+    def __init__(self, rapl_dir="/sys/class/powercap/intel-rapl/subsystem"):
         self._lin_rapl_dir = rapl_dir
         self._system = sys.platform.lower()
         self._rapl_files = []
@@ -275,6 +298,8 @@ def _fetch_rapl_files(self) -> None:
             with open(path) as f:
                 name = f.read().strip()
                 # Fake the name used by Power Gadget
+                # We ignore "core" in name as it seems to be included in "package" for Intel CPU.
+                # TODO: Use "dram" for memory power
                 if "package" in name:
                     name = f"Processor Energy Delta_{i}(kWh)"
                     i += 1
@@ -294,7 +319,7 @@ def _fetch_rapl_files(self) -> None:
                     logger.debug("We will read Intel RAPL files at %s", rapl_file)
                 except PermissionError as e:
                     raise PermissionError(
-                        "Unable to read Intel RAPL files for CPU power, we will use a constant for your CPU power."
+                        "PermissionError : Unable to read Intel RAPL files for CPU power, we will use a constant for your CPU power."
                         + " Please view https://github.com/mlco2/codecarbon/issues/244"
                         + " for workarounds : %s",
                         e,
@@ -332,8 +357,6 @@ def get_static_cpu_details(self) -> Dict:
         """
         Return CPU details without computing them.
         """
-        logger.debug("get_static_cpu_details %s", self._cpu_details)
-
         return self._cpu_details
 
     def start(self) -> None:
@@ -426,6 +449,18 @@ def _get_matching_cpu(
         start_cpu = model_raw.find(" CPU @ ")
         if start_cpu > 0:
             model_raw = model_raw[0:start_cpu]
+        model_raw = model_raw.replace(" CPU", "")
+        model_raw = re.sub(r" @\s*\d+\.\d+GHz", "", model_raw)
+        direct_match = process.extractOne(
+            model_raw,
+            cpu_df["Name"],
+            processor=lambda s: s.lower(),
+            scorer=fuzz.ratio,
+            score_cutoff=THRESHOLD_DIRECT,
+        )
+
+        if direct_match:
+            return direct_match[0]
         indirect_matches = process.extract(
             model_raw,
             cpu_df["Name"],
@@ -467,10 +502,18 @@ def _main(self) -> Tuple[str, int]:
                 + " Please contact us.",
                 cpu_model_detected,
             )
+            if is_psutil_available():
+                # Count thread of the CPU
+                threads = psutil.cpu_count(logical=True)
+                estimated_tdp = threads * DEFAULT_POWER_PER_CORE
+                logger.warning(
+                    f"We will use the default power consumption of {DEFAULT_POWER_PER_CORE} W per thread for your {threads} CPU, so {estimated_tdp}W."
+                )
+                return cpu_model_detected, estimated_tdp
             return cpu_model_detected, None
         logger.warning(
             "We were unable to detect your CPU using the `cpuinfo` package."
-            + " Resorting to a default power consumption of 85W."
+            + " Resorting to a default power consumption."
         )
         return "Unknown", None
 

codecarbon/core/emissions.py

@@ -90,7 +90,7 @@ def get_cloud_country_iso_code(self, cloud: CloudMetadata) -> str:
         selected = df.loc[flags]
         if not len(selected):
             raise ValueError(
-                "Unable to find country name for "
+                "Unable to find country ISO Code for "
                 f"cloud_provider={cloud.provider}, "
                 f"cloud_region={cloud.region}"
             )
@@ -105,7 +105,7 @@ def get_cloud_geo_region(self, cloud: CloudMetadata) -> str:
         selected = df.loc[flags]
         if not len(selected):
             raise ValueError(
-                "Unable to find country name for "
+                "Unable to find State/City name for "
                 f"cloud_provider={cloud.provider}, "
                 f"cloud_region={cloud.region}"
             )