From b2e5e3c20207583fcd2fbb7db5f149b0c3e9dd12 Mon Sep 17 00:00:00 2001
From: Christoph Alt <typ@ohnebild.com>
Date: Tue, 4 Oct 2022 18:31:22 +0200
Subject: [PATCH] added support for gitlab api
---
cbutil/__init__.py | 1 +
cbutil/get_job_info.py | 3 ++-
cbutil/gitlab_api.py | 53 ++++++++++++++++++++++++++++++++++++++++
tests/test_gitlab_api.py | 42 +++++++++++++++++++++++++++++++
4 files changed, 98 insertions(+), 1 deletion(-)
create mode 100644 cbutil/gitlab_api.py
create mode 100644 tests/test_gitlab_api.py
diff --git a/cbutil/__init__.py b/cbutil/__init__.py
index 8d2139f..6bd35ff 100644
--- a/cbutil/__init__.py
+++ b/cbutil/__init__.py
@@ -4,3 +4,4 @@ from .postprocessing import *
from .util import read_file_line_wise, time_conversion, file_time_to_sec, get_from_nested_dict
from .data_points import DataPoint, data_point_factory
from .get_job_info import get_url_from_env, get_job_datapoints
+from .gitlab_api import get_git_infos_from_api
diff --git a/cbutil/get_job_info.py b/cbutil/get_job_info.py
index 1a58d5a..7bb4db3 100644
--- a/cbutil/get_job_info.py
+++ b/cbutil/get_job_info.py
@@ -5,6 +5,7 @@ import dotenv
import requests
from cbutil.data_points import DataPoint, data_point_factory
+from cbutil.gitlab_api import add_pagination
logger = logging.getLogger(__file__)
@@ -35,7 +36,7 @@ def get_job_info(url: str):
for job in jobs.json():
yield job
if (next_page := jobs.headers['x-next-page']):
- next_url = f"{url}?page={next_page}"
+ next_url = add_pagination(url, next_page)
else:
break
diff --git a/cbutil/gitlab_api.py b/cbutil/gitlab_api.py
new file mode 100644
index 0000000..1aeab19
--- /dev/null
+++ b/cbutil/gitlab_api.py
@@ -0,0 +1,53 @@
+import requests
+from typing import Union
+import logging
+from cbutil.util import remove_newline
+
+logger = logging.getLogger(__file__)
+
+Project_id = Union[str, int]
+
+
+def url_encode(project_id: Project_id) -> str:
+ if isinstance(project_id, str):
+ return project_id.replace("/", "%2F")
+ else:
+ return project_id
+
+
+def add_pagination(url: str, page, *, per_page=20) -> str:
+ return f"{url}?per_page={per_page}&page={page}"
+
+
+def get_api_url(instance_url: str, project_id: Project_id) -> str:
+ return f"{instance_url}/api/v4/projects/{url_encode(project_id)}"
+
+
+def get_api_url_commits(base_api_url: str) -> str:
+ return f"{base_api_url}/repository/commits"
+
+
+def get_last_commit(instance_url: str, project_id: Project_id, branch: str = ''):
+ url = get_api_url_commits(get_api_url(instance_url, project_id))
+ url = add_pagination(url, page=1, per_page=1)
+ params = {'ref_name': branch if branch else 'master'}
+ commit = requests.get(url, params=params)
+ logger.info(f"requesting {url} with params")
+ if commit.status_code != 200:
+ commit.raise_for_status()
+ return commit.json()[0]
+
+
+def get_commit_infos_from_api(commit: dict) -> dict:
+ return commit['id'], remove_newline(commit['message'])
+
+
+def get_git_infos_from_api(instance_url, project_id: Project_id,
+ *,
+ branch='main',
+ commit_key="commit",
+ commit_msg_key="commit_message"):
+ last_commit = get_last_commit(instance_url, project_id, branch=branch)
+ commit, commit_msg = get_commit_infos_from_api(last_commit)
+ commit_msg = remove_newline(commit_msg)
+ return {commit_key: commit, commit_msg_key: commit_msg}
diff --git a/tests/test_gitlab_api.py b/tests/test_gitlab_api.py
new file mode 100644
index 0000000..4592870
--- /dev/null
+++ b/tests/test_gitlab_api.py
@@ -0,0 +1,42 @@
+from cbutil.gitlab_api import add_pagination, get_last_commit, get_git_infos_from_api, url_encode
+import os
+import pytest
+
+
+def test_url_encode():
+ assert 'walberla%2Fwalberla' == url_encode('walberla/walberla')
+ assert 'walberla%2Fwalberla' == url_encode('walberla%2Fwalberla')
+ assert 123 == url_encode(123)
+
+
+def test_pagination():
+ base_url = "http://base.url/"
+ assert add_pagination(base_url, 1) == f"{base_url}?per_page=20&page=1"
+ assert add_pagination(base_url, 1, per_page=1) == f"{base_url}?per_page=1&page=1"
+
+
+def get_local_infos(branch_name):
+ with os.popen(f'git rev-parse origin/{branch_name}') as rev:
+ expected = rev.read().strip()
+ return expected
+
+
+@pytest.mark.parametrize("project_id", ['ob28imeq%2Fcb-util', 976])
+def test_get_last_commit(project_id):
+ branch_name = 'main'
+ instance_url = 'https://i10git.cs.fau.de/'
+ expected = get_local_infos(branch_name)
+
+ last_commit = get_last_commit(instance_url, project_id, branch=branch_name)
+ assert last_commit['id'] == expected
+
+
+@pytest.mark.parametrize("project_id", ['ob28imeq%2Fcb-util', 976])
+def test_get_get_git_infos(project_id):
+ branch_name = 'main'
+ instance_url = 'https://i10git.cs.fau.de/'
+ expected = get_local_infos(branch_name)
+
+ git_infos = get_git_infos_from_api(instance_url, project_id)
+
+ assert git_infos['commit'] == expected
--
GitLab