Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
C
cb-util
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Christoph Alt
cb-util
Commits
d404c71c
Commit
d404c71c
authored
2 years ago
by
Christoph Alt
Browse files
Options
Downloads
Patches
Plain Diff
added util to read the job infos
parent
69a74514
No related branches found
No related tags found
No related merge requests found
Pipeline
#42911
passed
2 years ago
Stage: test
Stage: deploy
Changes
5
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
cbutil/__init__.py
+1
-0
1 addition, 0 deletions
cbutil/__init__.py
cbutil/data_points.py
+3
-2
3 additions, 2 deletions
cbutil/data_points.py
cbutil/get_job_info.py
+44
-0
44 additions, 0 deletions
cbutil/get_job_info.py
cbutil/util.py
+7
-2
7 additions, 2 deletions
cbutil/util.py
setup.py
+1
-0
1 addition, 0 deletions
setup.py
with
56 additions
and
4 deletions
cbutil/__init__.py
+
1
−
0
View file @
d404c71c
...
...
@@ -3,3 +3,4 @@ from .processing_functions import mesa_pd_text
from
.postprocessing
import
*
from
.util
import
read_file_line_wise
,
time_conversion
from
.data_points
import
DataPoint
,
data_point_factory
from
.get_job_info
import
get_url_from_env
,
get_job_datapoints
This diff is collapsed.
Click to expand it.
cbutil/data_points.py
+
3
−
2
View file @
d404c71c
from
dataclasses
import
dataclass
,
asdict
from
.util
import
time_conversion
import
logging
from
dataclasses
import
asdict
,
dataclass
from
.util
import
time_conversion
logger
=
logging
.
getLogger
(
__file__
)
...
...
This diff is collapsed.
Click to expand it.
cbutil/get_job_info.py
0 → 100644
+
44
−
0
View file @
d404c71c
import
logging
import
os
import
dotenv
import
requests
from
cbutil.data_points
import
DataPoint
,
data_point_factory
logger
=
logging
.
getLogger
(
__file__
)
def
load_from_env
(
env_path
=
"
.env
"
):
if
os
.
path
.
exists
(
env_path
):
dotenv
.
load_dotenv
(
env_path
)
return
os
.
environ
[
"
CI_API_V4_URL
"
],
os
.
environ
[
"
CI_PROJECT_ID
"
],
os
.
environ
[
"
CI_PIPELINE_ID
"
]
def
get_url_from_env
()
->
str
:
base_url
,
project_id
,
pipeline_id
=
load_from_env
()
logging
.
info
(
f
'
Loaded from env
{
base_url
}
{
project_id
}{
pipeline_id
}
'
)
return
get_api_url_pipelines
(
base_url
,
project_id
,
pipeline_id
)
def
get_api_url_pipelines
(
base_url
:
str
,
project_id
:
int
,
pipeline_id
:
int
):
return
f
"
{
base_url
}
/projects/
{
project_id
}
/pipelines/
{
pipeline_id
}
/jobs
"
def
get_job_info
(
url
:
str
):
jobs
=
requests
.
get
(
url
)
for
job
in
jobs
.
json
():
yield
job
def
create_job_datapoint
(
job
:
dict
)
->
DataPoint
:
return
data_point_factory
(
job
,
time_key
=
'
finished_at
'
,
measurement_name
=
"
JOB_INFOS
"
,
field_keys
=
[
'
duration
'
,
'
queued_duration
'
],
tag_keys
=
[
'
id
'
,
'
name
'
])
def
get_job_datapoints
(
url
):
for
job
in
get_job_info
(
url
):
yield
create_job_datapoint
(
job
)
This diff is collapsed.
Click to expand it.
cbutil/util.py
+
7
−
2
View file @
d404c71c
...
...
@@ -56,10 +56,15 @@ def file_time_to_sec(file_path) -> int:
return
int
(
os
.
path
.
getmtime
(
file_path
))
def
time_conversion
(
time_stamp
,
*
,
pattern
=
"
%Y-%m-%d %H:%M:%S
"
):
def
time_conversion
(
time_stamp
,
*
,
pattern
=
"
%Y-%m-%d %H:%M:%S
%z
"
):
try
:
return
int
(
time_stamp
)
except
ValueError
as
e
:
logger
.
exception
(
e
)
try
:
dt
=
datetime
.
strptime
(
time_stamp
,
pattern
)
except
ValueError
as
e
:
logger
.
exception
(
e
)
dt
=
datetime
.
fromisoformat
(
time_stamp
)
return
int
(
d
atetime
.
strptime
(
time_stamp
,
pattern
)
.
timestamp
())
return
int
(
d
t
.
timestamp
())
This diff is collapsed.
Click to expand it.
setup.py
+
1
−
0
View file @
d404c71c
...
...
@@ -14,6 +14,7 @@ setup(name="cb-util",
"
influxdb
"
,
"
gitpython
"
,
"
grafanalib
"
,
"
requests
"
,
],
setup_requires
=
[
'
pytest-runner
'
],
tests_require
=
[
'
pytest
'
]
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment