from __future__ import annotations
import re
import json
import os
import struct
from pathlib import Path
from typing import Any
import numpy as np
import tifffile
from tifffile import read_scanimage_metadata, matlabstr2py
from tifffile.tifffile import bytes2str, read_json, FileHandle
def _params_from_metadata_caiman(metadata):
"""
Generate parameters for CNMF from metadata.
Based on the pixel resolution and frame rate, the parameters are set to reasonable values.
Parameters
----------
metadata : dict
Metadata dictionary resulting from `lcp.get_metadata()`.
Returns
-------
dict
Dictionary of parameters for lbm_mc.
"""
params = _default_params_caiman()
if metadata is None:
print("No metadata found. Using default parameters.")
return params
params["main"]["fr"] = metadata["frame_rate"]
params["main"]["dxy"] = metadata["pixel_resolution"]
# typical neuron ~16 microns
gSig = round(16 / metadata["pixel_resolution"][0]) / 2
params["main"]["gSig"] = (int(gSig), int(gSig))
gSiz = (4 * gSig + 1, 4 * gSig + 1)
params["main"]["gSiz"] = gSiz
max_shifts = [int(round(10 / px)) for px in metadata["pixel_resolution"]]
params["main"]["max_shifts"] = max_shifts
strides = [int(round(64 / px)) for px in metadata["pixel_resolution"]]
params["main"]["strides"] = strides
# overlap should be ~neuron diameter
overlaps = [int(round(gSig / px)) for px in metadata["pixel_resolution"]]
if overlaps[0] < gSig:
print("Overlaps too small. Increasing to neuron diameter.")
overlaps = [int(gSig)] * 2
params["main"]["overlaps"] = overlaps
rf_0 = (strides[0] + overlaps[0]) // 2
rf_1 = (strides[1] + overlaps[1]) // 2
rf = int(np.mean([rf_0, rf_1]))
stride = int(np.mean([overlaps[0], overlaps[1]]))
params["main"]["rf"] = rf
params["main"]["stride"] = stride
return params
def _default_params_caiman():
"""
Default parameters for both registration and CNMF.
The exception is gSiz being set relative to gSig.
Returns
-------
dict
Dictionary of default parameter values for registration and segmentation.
Notes
-----
This will likely change as CaImAn is updated.
"""
gSig = 6
gSiz = (4 * gSig + 1, 4 * gSig + 1)
return {
"main": {
# Motion correction parameters
"pw_rigid": True,
"max_shifts": [6, 6],
"strides": [64, 64],
"overlaps": [8, 8],
"min_mov": None,
"gSig_filt": [0, 0],
"max_deviation_rigid": 3,
"border_nan": "copy",
"splits_els": 14,
"upsample_factor_grid": 4,
"use_cuda": False,
"num_frames_split": 50,
"niter_rig": 1,
"is3D": False,
"splits_rig": 14,
"num_splits_to_process_rig": None,
# CNMF parameters
"fr": 10,
"dxy": (1.0, 1.0),
"decay_time": 0.4,
"p": 2,
"nb": 3,
"K": 20,
"rf": 64,
"stride": [8, 8],
"gSig": gSig,
"gSiz": gSiz,
"method_init": "greedy_roi",
"rolling_sum": True,
"use_cnn": False,
"ssub": 1,
"tsub": 1,
"merge_thr": 0.7,
"bas_nonneg": True,
"min_SNR": 1.4,
"rval_thr": 0.8,
},
"refit": True,
}
def _params_from_metadata_suite2p(metadata, ops):
"""
Tau is 0.7 for GCaMP6f, 1.0 for GCaMP6m, 1.25-1.5 for GCaMP6s
"""
if metadata is None:
print("No metadata found. Using default parameters.")
return ops
# typical neuron ~16 microns
ops["fs"] = metadata["frame_rate"]
ops["nplanes"] = 1
ops["nchannels"] = 1
ops["do_bidiphase"] = 0
ops["do_regmetrics"] = True
# suite2p iterates each plane and takes ops['dxy'][i] where i is the plane index
ops["dx"] = [metadata["pixel_resolution"][0]]
ops["dy"] = [metadata["pixel_resolution"][1]]
return ops
def report_missing_metadata(file: os.PathLike | str):
tiff_file = tifffile.TiffFile(file)
if not tiff_file.software == "SI":
print(f"Missing SI software tag.")
if not tiff_file.description[:6] == "state.":
print(f"Missing 'state' software tag.")
if not "scanimage.SI" in tiff_file.description[-256:]:
print(f"Missing 'scanimage.SI' in description tag.")
def has_mbo_metadata(file: os.PathLike | str) -> bool:
"""
Check if a TIFF file has metadata from the Miller Brain Observatory.
Specifically, this checks for tiff_file.shaped_metadata, which is used to store system and user
supplied metadata.
Parameters
----------
file: os.PathLike
Path to the TIFF file.
Returns
-------
bool
True if the TIFF file has MBO metadata; False otherwise.
"""
if not file or not isinstance(file, (str, os.PathLike)):
raise ValueError(
"Invalid file path provided: must be a string or os.PathLike object."
f"Got: {file} of type {type(file)}"
)
# Tiffs
if Path(file).suffix in [".tif", ".tiff"]:
try:
tiff_file = tifffile.TiffFile(file)
if (
hasattr(tiff_file, "shaped_metadata")
and tiff_file.shaped_metadata is not None
):
return True
else:
return False
except Exception:
return False
return False
def is_raw_scanimage(file: os.PathLike | str) -> bool:
"""
Check if a TIFF file is a raw ScanImage TIFF.
Parameters
----------
file: os.PathLike
Path to the TIFF file.
Returns
-------
bool
True if the TIFF file is a raw ScanImage TIFF; False otherwise.
"""
if not file or not isinstance(file, (str, os.PathLike)):
return False
elif Path(file).suffix not in [".tif", ".tiff"]:
return False
try:
tiff_file = tifffile.TiffFile(file)
if (
# TiffFile.shaped_metadata is where we store metadata for processed tifs
# if this is not empty, we have a processed file
# otherwise, we have a raw scanimage tiff
hasattr(tiff_file, "shaped_metadata")
and tiff_file.shaped_metadata is not None
and isinstance(tiff_file.shaped_metadata, (list, tuple))
):
return False
else:
if tiff_file.scanimage_metadata is None:
print(f"No ScanImage metadata found in {file}.")
return False
return True
except Exception:
return False
def get_metadata_batch(files: list[os.PathLike | str], z_step=None, verbose=False):
"""
Extract and aggregate metadata from a list of TIFF files produced by ScanImage.
Parameters
----------
files : list of str or PathLike
List of paths to TIFF files.
z_step : float, optional
Z-step in microns to include in the returned metadata.
verbose : bool, optional
If True, include full metadata from the first TIFF in 'all' key.
Returns
-------
dict
Aggregated metadata dictionary with total frame count and per-file page counts.
"""
total_frames = 0
frame_indices = []
first_meta = None
for i, f in enumerate(files):
tf = tifffile.TiffFile(f)
num_pages = len(tf.pages)
frame_indices.append(num_pages)
total_frames += num_pages
if i == 0:
if not is_raw_scanimage(f):
base = tf.shaped_metadata[0]["image"]
elif (
hasattr(tf, "scanimage_metadata") and tf.scanimage_metadata is not None
):
base = get_metadata(f, z_step=z_step, verbose=verbose)
else:
raise ValueError(f"No metadata found in {f}.")
first_meta = base.copy()
first_meta["num_frames"] = total_frames
first_meta["frame_indices"] = frame_indices
return first_meta
def params_from_metadata(metadata, base_ops, pipeline="suite2p"):
"""
Use metadata to get sensible default pipeline parameters.
If ops are not provided, uses suite2p.default_ops(). Sets framerate, pixel resolution, and do_metrics=True.
Parameters
----------
metadata : dict
Result of mbo.get_metadata()
base_ops : dict
Ops dict to use as a base.
pipeline : str, optional
The pipeline to use. Default is "suite2p".
"""
if pipeline.lower() == "caiman":
print("Warning: CaImAn is not stable, proceed at your own risk.")
return _params_from_metadata_caiman(metadata)
elif pipeline.lower() == "suite2p":
print("Setting pipeline to suite2p")
return _params_from_metadata_suite2p(metadata, base_ops)
else:
raise ValueError(
f"Pipeline {pipeline} not recognized. Use 'caiman' or 'suite2'"
)
def read_scanimage_metadata_tifffile(
fh: FileHandle, /
) -> tuple[dict[str, Any], dict[str, Any], int]:
"""FROM TIFFFILE for DEVELOPMENT
Read ScanImage BigTIFF v3 or v4 static and ROI metadata from file.
The settings can be used to read image and metadata without parsing
the TIFF file.
Frame data and ROI groups can alternatively be obtained from the Software
and Artist tags of any TIFF page.
Parameters:
fh: Binary file handle to read from.
Returns:
- Non-varying frame data, parsed with :py:func:`matlabstr2py`.
- ROI group data, parsed from JSON.
- Version of metadata (3 or 4).
Raises:
ValueError: File does not contain valid ScanImage metadata.
"""
fh.seek(0)
try:
byteorder, version = struct.unpack("<2sH", fh.read(4))
if byteorder != b"II" or version != 43:
raise ValueError("not a BigTIFF file")
fh.seek(16)
magic, version, size0, size1 = struct.unpack("<IIII", fh.read(16))
if magic != 117637889 or version not in {3, 4}:
raise ValueError(f"invalid magic {magic} or version {version} number")
except UnicodeDecodeError as exc:
raise ValueError("file must be opened in binary mode") from exc
except Exception as exc:
raise ValueError("not a ScanImage BigTIFF v3 or v4 file") from exc
frame_data = matlabstr2py(bytes2str(fh.read(size0)[:-1]))
roi_data = read_json(fh, "<", 0, size1, 0) if size1 > 1 else {}
return frame_data, roi_data, version
def matlabstr(obj):
"""Convert Python dict to ScanImage-style MATLAB string."""
def _format(v):
if isinstance(v, list):
if all(isinstance(i, str) for i in v):
return "{" + " ".join(f"'{i}'" for i in v) + "}"
return "[" + " ".join(str(i) for i in v) + "]"
if isinstance(v, str):
return f"'{v}'"
if isinstance(v, bool):
return "true" if v else "false"
return str(v)
return "\n".join(f"{k} = {_format(v)}" for k, v in obj.items())
def _parse_value(value_str):
if value_str.startswith("'") and value_str.endswith("'"):
return value_str[1:-1]
if value_str == "true":
return True
if value_str == "false":
return False
if value_str == "NaN":
return float("nan")
if value_str == "Inf":
return float("inf")
if re.match(r"^\d+(\.\d+)?$", value_str):
return float(value_str) if "." in value_str else int(value_str)
if re.match(r"^\[(.*)]$", value_str):
return [_parse_value(v.strip()) for v in value_str[1:-1].split()]
return value_str
def _parse_key_value(parse_line):
key_str, value_str = parse_line.split(" = ", 1)
return key_str, _parse_value(value_str)
def parse(metadata_str):
"""
Parses the metadata string from a ScanImage Tiff file.
:param metadata_str:
:return metadata_kv, metadata_json:
"""
lines = metadata_str.split("\n")
metadata_kv = {}
json_portion = []
parsing_json = False
for line in lines:
line = line.strip()
if not line:
continue
if line.startswith("SI."):
key, value = _parse_key_value(line)
metadata_kv[key] = value
elif line.startswith("{"):
parsing_json = True
if parsing_json:
json_portion.append(line)
metadata_json = json.loads("\n".join(json_portion))
return metadata_kv, metadata_json
def find_scanimage_metadata(path):
with tifffile.TiffFile(path) as tif:
if hasattr(tif, "scanimage_metadata"):
return tif.scanimage_metadata
p = tif.pages[0]
cand = []
for tag in ("ImageDescription", "Software"):
if tag in p.tags:
cand.append(p.tags[tag].value)
if getattr(p, "description", None):
cand.append(p.description)
cand.extend(str(tif.__dict__.get(k, "")) for k in tif.__dict__)
for s in cand:
if isinstance(s, bytes):
s = s.decode(errors="ignore")
m = re.search(r"{.*ScanImage.*}", s, re.S)
if m:
try:
return json.loads(m.group(0))
except Exception:
return m.group(0)
return None
