added support for gitlab api

cbutil/__init__.py

@@ -4,3 +4,4 @@ from .postprocessing import *
 from .util import read_file_line_wise, time_conversion, file_time_to_sec, get_from_nested_dict
 from .data_points import DataPoint, data_point_factory
 from .get_job_info import get_url_from_env, get_job_datapoints
+from .gitlab_api import get_git_infos_from_api

cbutil/get_job_info.py

@@ -5,6 +5,7 @@ import dotenv
 import requests
 
 from cbutil.data_points import DataPoint, data_point_factory
+from cbutil.gitlab_api import add_pagination
 
 logger = logging.getLogger(__file__)
 
@@ -35,7 +36,7 @@ def get_job_info(url: str):
         for job in jobs.json():
             yield job
         if (next_page := jobs.headers['x-next-page']):
-            next_url = f"{url}?page={next_page}"
+            next_url = add_pagination(url, next_page)
         else:
             break
 

cbutil/gitlab_api.py

+import requests
+from typing import Union
+import logging
+from cbutil.util import remove_newline
+
+logger = logging.getLogger(__file__)
+
+Project_id = Union[str, int]
+
+
+def url_encode(project_id: Project_id) -> str:
+    if isinstance(project_id, str):
+        return project_id.replace("/", "%2F")
+    else:
+        return project_id
+
+
+def add_pagination(url: str, page, *, per_page=20) -> str:
+    return f"{url}?per_page={per_page}&page={page}"
+
+
+def get_api_url(instance_url: str, project_id: Project_id) -> str:
+    return f"{instance_url}/api/v4/projects/{url_encode(project_id)}"
+
+
+def get_api_url_commits(base_api_url: str) -> str:
+    return f"{base_api_url}/repository/commits"
+
+
+def get_last_commit(instance_url: str, project_id: Project_id, branch: str = ''):
+    url = get_api_url_commits(get_api_url(instance_url, project_id))
+    url = add_pagination(url, page=1, per_page=1)
+    params = {'ref_name': branch if branch else 'master'}
+    commit = requests.get(url, params=params)
+    logger.info(f"requesting {url} with params")
+    if commit.status_code != 200:
+        commit.raise_for_status()
+    return commit.json()[0]
+
+
+def get_commit_infos_from_api(commit: dict) -> dict:
+    return commit['id'], remove_newline(commit['message'])
+
+
+def get_git_infos_from_api(instance_url, project_id: Project_id,
+                           *,
+                           branch='main',
+                           commit_key="commit",
+                           commit_msg_key="commit_message"):
+    last_commit = get_last_commit(instance_url, project_id, branch=branch)
+    commit, commit_msg = get_commit_infos_from_api(last_commit)
+    commit_msg = remove_newline(commit_msg)
+    return {commit_key: commit, commit_msg_key: commit_msg}

tests/test_gitlab_api.py

+from cbutil.gitlab_api import add_pagination, get_last_commit, get_git_infos_from_api, url_encode
+import os
+import pytest
+
+
+def test_url_encode():
+    assert 'walberla%2Fwalberla' == url_encode('walberla/walberla')
+    assert 'walberla%2Fwalberla' == url_encode('walberla%2Fwalberla')
+    assert 123 == url_encode(123)
+
+
+def test_pagination():
+    base_url = "http://base.url/"
+    assert add_pagination(base_url, 1) == f"{base_url}?per_page=20&page=1"
+    assert add_pagination(base_url, 1, per_page=1) == f"{base_url}?per_page=1&page=1"
+
+
+def get_local_infos(branch_name):
+    with os.popen(f'git rev-parse origin/{branch_name}') as rev:
+        expected = rev.read().strip()
+    return expected
+
+
+@pytest.mark.parametrize("project_id", ['ob28imeq%2Fcb-util', 976])
+def test_get_last_commit(project_id):
+    branch_name = 'main'
+    instance_url = 'https://i10git.cs.fau.de/'
+    expected = get_local_infos(branch_name)
+
+    last_commit = get_last_commit(instance_url, project_id, branch=branch_name)
+    assert last_commit['id'] == expected
+
+
+@pytest.mark.parametrize("project_id", ['ob28imeq%2Fcb-util', 976])
+def test_get_get_git_infos(project_id):
+    branch_name = 'main'
+    instance_url = 'https://i10git.cs.fau.de/'
+    expected = get_local_infos(branch_name)
+
+    git_infos = get_git_infos_from_api(instance_url, project_id)
+
+    assert git_infos['commit'] == expected