Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
A
alibylite
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Deploy
Releases
Package Registry
Container Registry
Model registry
Operate
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Swain Lab
aliby
alibylite
Commits
967d8e74
Commit
967d8e74
authored
3 years ago
by
Alán Muñoz
Browse files
Options
Downloads
Patches
Plain Diff
bugfix fringe positions using dif channels
parent
949389dd
No related branches found
Branches containing commit
No related tags found
Tags containing commit
No related merge requests found
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
aliby/io/metadata_parser.py
+28
-20
28 additions, 20 deletions
aliby/io/metadata_parser.py
with
28 additions
and
20 deletions
aliby/io/metadata_parser.py
+
28
−
20
View file @
967d8e74
...
@@ -15,21 +15,23 @@ from logfile_parser import Parser
...
@@ -15,21 +15,23 @@ from logfile_parser import Parser
# then pare down on what specific information is really useful later.
# then pare down on what specific information is really useful later.
# Needed because HDF5 attributes do not support dictionaries
# Needed because HDF5 attributes do not support dictionaries
def
flatten_dict
(
nested_dict
,
separator
=
'
/
'
):
def
flatten_dict
(
nested_dict
,
separator
=
"
/
"
):
'''
"""
Flattens nested dictionary
Flattens nested dictionary
'''
"""
df
=
pd
.
json_normalize
(
nested_dict
,
sep
=
separator
)
df
=
pd
.
json_normalize
(
nested_dict
,
sep
=
separator
)
return
df
.
to_dict
(
orient
=
'
records
'
)[
0
]
return
df
.
to_dict
(
orient
=
"
records
"
)[
0
]
# Needed because HDF5 attributes do not support datetime objects
# Needed because HDF5 attributes do not support datetime objects
# Takes care of time zones & daylight saving
# Takes care of time zones & daylight saving
def
datetime_to_timestamp
(
time
,
locale
=
'
Europe/London
'
):
def
datetime_to_timestamp
(
time
,
locale
=
"
Europe/London
"
):
'''
"""
Convert datetime object to UNIX timestamp
Convert datetime object to UNIX timestamp
'''
"""
return
timezone
(
locale
).
localize
(
time
).
timestamp
()
return
timezone
(
locale
).
localize
(
time
).
timestamp
()
def
find_file
(
root_dir
,
regex
):
def
find_file
(
root_dir
,
regex
):
file
=
glob
.
glob
(
os
.
path
.
join
(
str
(
root_dir
),
regex
))
file
=
glob
.
glob
(
os
.
path
.
join
(
str
(
root_dir
),
regex
))
if
len
(
file
)
!=
1
:
if
len
(
file
)
!=
1
:
...
@@ -37,33 +39,36 @@ def find_file(root_dir, regex):
...
@@ -37,33 +39,36 @@ def find_file(root_dir, regex):
else
:
else
:
return
file
[
0
]
return
file
[
0
]
# TODO: re-write this as a class if appropriate
# TODO: re-write this as a class if appropriate
# WARNING: grammars depend on the directory structure of a locally installed
# WARNING: grammars depend on the directory structure of a locally installed
# logfile_parser repo
# logfile_parser repo
def
parse_logfiles
(
root_dir
,
def
parse_logfiles
(
acq_grammar
=
'
multiDGUI_acq_format.json
'
,
root_dir
,
log_grammar
=
'
multiDGUI_log_format.json
'
):
acq_grammar
=
"
multiDGUI_acq_format.json
"
,
'''
log_grammar
=
"
multiDGUI_log_format.json
"
,
):
"""
Parse acq and log files depending on the grammar specified, then merge into
Parse acq and log files depending on the grammar specified, then merge into
single dict.
single dict.
'''
"""
# Both acq and log files contain useful information.
# Both acq and log files contain useful information.
#ACQ_FILE = 'flavin_htb2_glucose_long_ramp_DelftAcq.txt'
#
ACQ_FILE = 'flavin_htb2_glucose_long_ramp_DelftAcq.txt'
#LOG_FILE = 'flavin_htb2_glucose_long_ramp_Delftlog.txt'
#
LOG_FILE = 'flavin_htb2_glucose_long_ramp_Delftlog.txt'
log_parser
=
Parser
(
log_grammar
)
log_parser
=
Parser
(
log_grammar
)
try
:
try
:
log_file
=
find_file
(
root_dir
,
'
*log.txt
'
)
log_file
=
find_file
(
root_dir
,
"
*log.txt
"
)
except
:
except
:
raise
ValueError
(
'
Experiment log file not found.
'
)
raise
ValueError
(
"
Experiment log file not found.
"
)
with
open
(
log_file
,
'
r
'
)
as
f
:
with
open
(
log_file
,
"
r
"
)
as
f
:
log_parsed
=
log_parser
.
parse
(
f
)
log_parsed
=
log_parser
.
parse
(
f
)
acq_parser
=
Parser
(
acq_grammar
)
acq_parser
=
Parser
(
acq_grammar
)
try
:
try
:
acq_file
=
find_file
(
root_dir
,
'
*[Aa]cq.txt
'
)
acq_file
=
find_file
(
root_dir
,
"
*[Aa]cq.txt
"
)
except
:
except
:
raise
ValueError
(
'
Experiment acq file not found.
'
)
raise
ValueError
(
"
Experiment acq file not found.
"
)
with
open
(
acq_file
,
'
r
'
)
as
f
:
with
open
(
acq_file
,
"
r
"
)
as
f
:
acq_parsed
=
acq_parser
.
parse
(
f
)
acq_parsed
=
acq_parser
.
parse
(
f
)
parsed
=
{
**
acq_parsed
,
**
log_parsed
}
parsed
=
{
**
acq_parsed
,
**
log_parsed
}
...
@@ -73,5 +78,8 @@ def parse_logfiles(root_dir,
...
@@ -73,5 +78,8 @@ def parse_logfiles(root_dir,
parsed
[
key
]
=
datetime_to_timestamp
(
value
)
parsed
[
key
]
=
datetime_to_timestamp
(
value
)
parsed_flattened
=
flatten_dict
(
parsed
)
parsed_flattened
=
flatten_dict
(
parsed
)
for
k
,
v
in
parsed_flattened
.
items
():
if
isinstance
(
v
,
list
):
parsed_flattened
[
k
]
=
[
0
if
el
is
None
else
el
for
el
in
v
]
return
parsed_flattened
return
parsed_flattened
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment