import re
from logging.config import dictConfig
from typing import List
import pandas as pd
from pandas import DataFrame
from musif.config import (
DELETE_SOUND_COLUMNS,
ENDSWITH,
INSTRUMENTS_TO_DELETE,
INSTRUMENTS_TO_KEEP,
STARTSWITH,
)
from musif.extract.basic_modules.scoring.constants import (
FAMILY_INSTRUMENTATION,
FAMILY_SCORING,
)
from musif.extract.features.harmony.constants import (
KEY_MODULATORY,
KEY_PERCENTAGE,
KEY_PREFIX,
)
from musif.extract.features.prefix import get_part_prefix
from musif.extract.features.scale.constants import DEGREE_PREFIX
from musif.logs import pinfo
from .constants import voices_list_prefixes
[docs]
def join_part_degrees(
total_degrees: List[str], part: str, df: DataFrame, sufix: str = ""
) -> None:
"""Group one prefix's per-degree columns into Asc/Desc/Dasc/Ddesc/Nat/Nonat
aggregates, separately for _Count and _Per columns.
The accidental is matched as a token of the column name (not a substring),
so prefixes containing 'b' or 'x' (PartOb_, PartTbn_, ...) cannot
contaminate the groups.
"""
pattern = re.compile(
re.escape(part) + r"Degree(b+|#x?|x)?(\d+)_(Count|Per)" + re.escape(sufix) + "$"
)
for kind in ("Count", "Per"):
selected = {name: [] for name in ("Asc", "Desc", "Dasc", "Ddesc", "Nat", "Nonat")}
for col in total_degrees:
match = pattern.fullmatch(col)
if match is None or match.group(3) != kind:
continue
accidental = match.group(1) or ""
if accidental == "":
selected["Nat"].append(col)
else:
selected["Nonat"].append(col)
if accidental == "#":
selected["Asc"].append(col)
elif accidental == "b":
selected["Desc"].append(col)
elif accidental in ("x", "#x"):
selected["Dasc"].append(col)
else: # bb, bbb, ...
selected["Ddesc"].append(col)
if not any(selected.values()):
continue
# where the part has no degree value at all (it is not in the score,
# or the degrees could not be computed) its groups stay empty too
measured = df[[col for cols in selected.values() for col in cols]].notna().any(axis=1)
for name, cols in selected.items():
column = part + DEGREE_PREFIX + "_" + name + "_" + kind + sufix
total = df[cols].sum(axis=1) if cols else pd.Series(0.0, index=df.index)
df[column] = total.where(measured)
def log_errors_and_shape(
composer_counter: list, novoices_counter: list, df: DataFrame
) -> None:
pinfo(f"\nTotal files skipped by composer: {len(composer_counter)}")
pinfo(str(composer_counter))
pinfo(f"\nTotal files skipped by no-voices: { len(novoices_counter)}")
pinfo(str(novoices_counter))
# pinfo(f"\nTotal files skipped by duetos/trietos: {len(duetos_counter)}")
# pinfo(str(duetos_counter))
pinfo(f"\nFinal shape of the DataFrame: {df.shape[0]} rows, {df.shape[1]} features")
def _delete_columns(data: DataFrame, config: dictConfig) -> None:
# pinfo("\nDeleting not useful columns...")
to_delete = []
instruments_to_keep = [get_part_prefix(i) for i in config[INSTRUMENTS_TO_KEEP]]
for inst in config[INSTRUMENTS_TO_DELETE]:
# for i in data.columns:
# if "Part" + inst + "_" in i
part_prefix = "Part" + inst # + "_"
for col in data.columns:
if part_prefix in col and all(
inst not in col for inst in instruments_to_keep
):
pass
to_delete.append(col)
else:
pass
# to_delete += [i for i in data.columns if part_prefix in i and instrument not in i for instrument in instruments_to_keep]
to_delete += [i for i in data.columns if i.endswith(tuple(config[ENDSWITH]))]
to_delete += [i for i in data.columns if i.startswith(tuple(config[STARTSWITH]))]
to_delete += [
col
for col in data.columns
if any(substring in col for substring in config["columns_contain"])
]
to_delete += [
col
for col in data.columns
if any(string == col for string in config["columns_match"])
]
if config.get(DELETE_SOUND_COLUMNS, False):
to_delete += [
i for i in data.columns if i.startswith("Sound") and "Voice" not in i
]
to_delete += [FAMILY_INSTRUMENTATION, FAMILY_SCORING]
# Remove empty voices
to_delete += [
col
for col in data.columns
if col.startswith(tuple(voices_list_prefixes))
and all(data[col].isnull().values)
]
# removing columns containing nans; None means "keep all"
if config['delete_columns_with_nans']:
th = config["max_nan_columns"]
th = 1.0 if th is None else th
idx = data.isna().sum(axis=0) / data.shape[0] > th
to_delete += data.columns[idx].to_list()
data.drop(columns=to_delete, inplace=True, errors="ignore")
def join_keys(df: DataFrame) -> None:
key_SD = [
i
for i in [
KEY_PREFIX + "IV" + KEY_PERCENTAGE,
KEY_PREFIX + "II" + KEY_PERCENTAGE,
KEY_PREFIX + "VI" + KEY_PERCENTAGE,
]
if i in df
]
key_sd = [
i
for i in [
KEY_PREFIX + "iv" + KEY_PERCENTAGE,
KEY_PREFIX + "ii" + KEY_PERCENTAGE,
]
if i in df
]
key_tonic = [
i
for i in [KEY_PREFIX + "I" + KEY_PERCENTAGE, KEY_PREFIX + "i" + KEY_PERCENTAGE]
if i in df
]
key_rel = [
i
for i in [
KEY_PREFIX + "III" + KEY_PERCENTAGE,
KEY_PREFIX + "vi" + KEY_PERCENTAGE,
]
if i in df
]
key_D = [i for i in [KEY_PREFIX + "V" + KEY_PERCENTAGE] if i in df]
key_d = [i for i in [KEY_PREFIX + "v" + KEY_PERCENTAGE] if i in df]
total_key = key_rel + key_tonic + key_sd + key_SD + key_D + key_d
derived = {
KEY_PREFIX + name + KEY_PERCENTAGE
for name in ("SD", "sd", "SubD", "D", "d", "Dom", "T", "rel", "Other")
}
others_key = [
i
for i in df.columns
if KEY_PREFIX in i
and i not in total_key
and KEY_MODULATORY not in i
and i not in derived
]
df[KEY_PREFIX + "SD" + KEY_PERCENTAGE] = df[key_SD].sum(axis=1)
df[KEY_PREFIX + "sd" + KEY_PERCENTAGE] = df[key_sd].sum(axis=1)
df[KEY_PREFIX + "SubD" + KEY_PERCENTAGE] = (
df[KEY_PREFIX + "sd" + KEY_PERCENTAGE] + df[KEY_PREFIX + "SD" + KEY_PERCENTAGE]
)
df[KEY_PREFIX + "D" + KEY_PERCENTAGE] = df[key_D].sum(axis=1)
df[KEY_PREFIX + "d" + KEY_PERCENTAGE] = df[key_d].sum(axis=1)
df[KEY_PREFIX + "Dom" + KEY_PERCENTAGE] = (
df[KEY_PREFIX + "D" + KEY_PERCENTAGE] + df[KEY_PREFIX + "d" + KEY_PERCENTAGE]
)
df[KEY_PREFIX + "T" + KEY_PERCENTAGE] = df[key_tonic].sum(axis=1)
df[KEY_PREFIX + "rel" + KEY_PERCENTAGE] = df[key_rel].sum(axis=1)
df[KEY_PREFIX + "Other" + KEY_PERCENTAGE] = df[others_key].sum(axis=1)
# df.drop(total_key + others_key, axis = 1, inplace=True)
def join_keys_modulatory(df: DataFrame):
key_SD = [
i
for i in [
KEY_PREFIX + KEY_MODULATORY + "IV",
KEY_PREFIX + KEY_MODULATORY + "II",
KEY_PREFIX + KEY_MODULATORY + "VI",
]
if i in df
]
key_sd = [
i
for i in [
KEY_PREFIX + KEY_MODULATORY + "iv",
KEY_PREFIX + KEY_MODULATORY + "ii",
]
if i in df
]
key_tonic = [
i
for i in [KEY_PREFIX + KEY_MODULATORY + "I", KEY_PREFIX + KEY_MODULATORY + "i"]
if i in df
]
key_rel = [
i
for i in [
KEY_PREFIX + KEY_MODULATORY + "III",
KEY_PREFIX + KEY_MODULATORY + "vi",
]
if i in df
]
key_D = [i for i in [KEY_PREFIX + KEY_MODULATORY + "V"] if i in df]
key_d = [i for i in [KEY_PREFIX + KEY_MODULATORY + "v"] if i in df]
total_key_mod = key_rel + key_tonic + key_sd + key_SD + key_D + key_d
derived_mod = {
KEY_PREFIX + KEY_MODULATORY + name
for name in ("SD", "sd", "SubD", "D", "d", "Dom", "T", "rel", "Other")
}
others_key_mod = [
i
for i in df.columns
if KEY_PREFIX + KEY_MODULATORY in i
and i not in total_key_mod
and i not in derived_mod
]
df[KEY_PREFIX + KEY_MODULATORY + "SD"] = df[key_SD].sum(axis=1)
df[KEY_PREFIX + KEY_MODULATORY + "sd"] = df[key_sd].sum(axis=1)
df[KEY_PREFIX + KEY_MODULATORY + "SubD"] = (
df[KEY_PREFIX + KEY_MODULATORY + "sd"] + df[KEY_PREFIX + KEY_MODULATORY + "SD"]
)
df[KEY_PREFIX + KEY_MODULATORY + "D"] = df[key_D].sum(axis=1)
df[KEY_PREFIX + KEY_MODULATORY + "d"] = df[key_d].sum(axis=1)
df[KEY_PREFIX + KEY_MODULATORY + "Dom"] = (
df[KEY_PREFIX + KEY_MODULATORY + "D"] + df[KEY_PREFIX + KEY_MODULATORY + "d"]
)
df[KEY_PREFIX + KEY_MODULATORY + "T"] = df[key_tonic].sum(axis=1)
df[KEY_PREFIX + KEY_MODULATORY + "rel"] = df[key_rel].sum(axis=1)
df[KEY_PREFIX + KEY_MODULATORY + "Other"] = df[others_key_mod].sum(axis=1)
def _drop_filenames_nan_rows(df):
rows_with_nan_filename = list(df[df['FileName'].isna()]['FileName'].index)
if len(rows_with_nan_filename)>0:
print('There are som files with computation errors!')
print(rows_with_nan_filename)
df.dropna(subset=['FileName'], inplace=True)
[docs]
def merge_dataframes(name: str, dest_path: str) -> None:
"""
Takes two dataframes and joins them, apart from deleting rows that are all nans.
This is intended for cases where all extraction of a folder cannot be done all at once.
Returns
------
Dataframe with the extracted features as a concatenation of two dataframes
"""
csv = ".csv"
name1 = name + "_1" + csv
name2 = name + "_2" + csv
df1 = pd.read_csv(name1, low_memory=False)
df2 = pd.read_csv(name2, low_memory=False)
_drop_filenames_nan_rows(df1)
_drop_filenames_nan_rows(df2)
total_dataframe = pd.concat((df1, df2), axis=0)
total_dataframe.to_csv(dest_path + csv, index=False)