Source code for corpustools.corpus.classes.lexicon

from corpustools import __version__ as currentPCTversion
from decimal import Decimal
import regex as re
import random
import collections
import operator
import locale
import copy

from corpustools.exceptions import CorpusIntegrityError

SPECIAL_SYMBOL_RE = ['.', '^', '$', '*', '+', '?', '|', '{', '}', '[', ']', '#', '(', ')', '\'', '\"']

[docs]class Segment(object): """ Class for segment symbols Parameters ---------- symbol : str Segment symbol Attributes ---------- _features : dict Feature specification for the segment """ def __init__(self, symbol, features=None): #None defaults are for word-boundary symbols self.symbol = symbol if features is None: self.features = {} else: self.features = features def specify(self, feature_dict): """ Specify a segment with a new feature specification Parameters ---------- feature_dict : dict Feature specification """ self.features = {k.lower(): v for k,v in feature_dict.items()} def minimal_difference(self, other, features): """ Check if this segment is a minimal feature difference with another segment (ignoring some _features) Parameters ---------- other : Segment Segment to compare with _features : list Features that are allowed to vary between the two segments Returns ------- bool True if all _features other than the specified ones match, False otherwise """ for k, v in self.features.items(): if k in features: continue if v != other[k]: return False return True def feature_match(self, specification): """ Return true if segment matches specification, false otherwise. Parameters ---------- specification : object Specification can be a single feature value '+feature', a list of feature values ['+feature1','-feature2'], or a dictionary of features and values {'feature1': '+', 'feature2': '-'} Returns ------- bool True if this segment contains the feature values in the specification """ if isinstance(specification,str): try: if self[specification[1:]]!=specification[0]: return False except KeyError: return False elif isinstance(specification,list) or isinstance(specification, set): for f in specification: try: if self[f[1:]]!=f[0]: return False except KeyError: return False elif isinstance(specification, dict): for f,v in specification.items(): try: if self[f] != v: return False except KeyError: return False, return True def __contains__(self, item): return item.lower() in self.features def __getitem__(self, key): #return self.features[key.lower()] return self.features[key] def __setitem__(self, key, value): #self.features[key.lower()] = value self.features[key] = value def __repr__(self): return self.__str__() def __str__(self): return self.symbol def __eq__(self, other): """Two segments are considered equal if their symbol attributes match """ if isinstance(other, Segment): return self.symbol == other.symbol else: return self.symbol == other def __ne__(self, other): return not self.__eq__(other) def __lt__(self,other): if isinstance(other, Segment): return self.symbol < other.symbol else: return self.symbol < other def __le__(self,other): return (self.symbol == other.symbol or self.symbol < other.symbol) def __ge__(self,other): return (self.symbol == other.symbol or self.symbol > other.symbol) def __gt__(self,other): if isinstance(other, Segment): return self.symbol > other.symbol else: return self.symbol > other def __len__(self): return len(self.symbol)
[docs]class Transcription(object): """ Transcription object, sequence of symbols Parameters ---------- seg_list : list List of segments that form the transcription. Elements in the list, can be Segments, strings, or BaseAnnotations Attributes ---------- _list : list List of strings representing segment symbols stress_pattern: dict Dictionary with keys of segment indices and values of the stress for that segment boundaries : dict Possible keys of 'morpheme' or 'tone' that keeps track of where morpheme or tone boundaries are inserted """ def __init__(self, seg_list): # Note to myself: in the original implementation, seg_list is a list # of BaseAnnotation type # If there is a syllable delimiter, then it's a list of SyllableBaseAnnotation self._syllable_list = [] # a list of dictionaries self._list = [] self.stress_pattern = {} self.boundaries = {} # TODO: Don't know when this is used cur_group = 0 cur_tone = None if seg_list: first_element = seg_list[0] try: if first_element.type == "syllable": has_syllable_level = True else: has_syllable_level = False except AttributeError: has_syllable_level = False if has_syllable_level: # TODO: need to extend the function to be able to add new words in the future for base_syll in seg_list: self._syllable_list.append(base_syll) for i, s in enumerate(base_syll): try: self._list.append(s.label) # if s.begin is not None and s.end is not None: # self._times.append((s.begin,s.end)) if s.stress is not None: self.stress_pattern[i] = s.stress if s.tone is not None: if 'tone' not in self.boundaries: self.boundaries['tone'] = {} if s.tone != cur_tone: self.boundaries['tone'][i] = s.tone cur_tone = s.tone if s.group is not None: if 'morpheme' not in self.boundaries: self.boundaries['morpheme'] = [] if s.group != cur_group: self.boundaries['morpheme'].append(i) cur_group = s.group except AttributeError: if isinstance(s, str) or isinstance(s, Segment): self._list.append(s) elif isinstance(s, dict): try: symbol = s['label'] except KeyError: symbol = s['symbol'] self._list.append(symbol) # if 'begin' in s and 'end' in s: # self._times.append((s['begin'],s['end'])) elif isinstance(s, list): if len(s) == 3: # TODO: what is this line for? self._list.append(s[0]) # self._times.append((s[1],s[2])) else: raise (NotImplementedError('That format for seg_list is not supported.')) else: raise (NotImplementedError('That format for seg_list is not supported.')) else: for i, s in enumerate(seg_list): try: self._list.append(s.label) #if s.begin is not None and s.end is not None: # self._times.append((s.begin,s.end)) if s.stress is not None: self.stress_pattern[i] = s.stress if s.tone is not None: if 'tone' not in self.boundaries: self.boundaries['tone'] = {} if s.tone != cur_tone: self.boundaries['tone'][i] = s.tone cur_tone = s.tone if s.group is not None: if 'morpheme' not in self.boundaries: self.boundaries['morpheme'] = [] if s.group != cur_group: self.boundaries['morpheme'].append(i) cur_group = s.group except AttributeError: if isinstance(s, str) or isinstance(s, Segment): self._list.append(s) elif isinstance(s, dict): try: symbol = s['label'] except KeyError: symbol = s['symbol'] self._list.append(symbol) #if 'begin' in s and 'end' in s: # self._times.append((s['begin'],s['end'])) elif isinstance(s, list): if len(s) == 3: # TODO: what is this line for? self._list.append(s[0]) #self._times.append((s[1],s[2])) else: raise(NotImplementedError('That format for seg_list is not supported.')) else: raise(NotImplementedError('That format for seg_list is not supported.')) def with_word_boundaries(self): """ Return the string of segments with word boundaries surrounding them Returns ------- list Transcription with word boundaries """ return ['#'] + self._list + ['#'] def with_syllable_and_word_boundaries(self): syllable_str = '' for syllable in self._syllable_list: onset_str = '' for seg in syllable.onset: onset_str += seg.label onset_str = '.' + onset_str + '.' nucleus_str = '' for seg in syllable.nucleus: nucleus_str += seg.label nucleus_str = '.' + nucleus_str + '.' coda_str = '' for seg in syllable.coda: coda_str += seg.label coda_str = '.' + coda_str + '.' if syllable.stress: stress_str = syllable.stress else: stress_str = '' if syllable.tone: tone_str = syllable.tone else: tone_str = '' syllable_str += '-' + stress_str + onset_str + nucleus_str + coda_str + tone_str + '-' #print('-#-' + syllable_str + '-#-') return '-#-' + syllable_str + '-#-' def find(self, environment, mode='segMode'): """ Find instances of an EnvironmentFilter in the Transcription Most notably, this function is called to perform phonological searches Parameters ---------- environment : EnvironmentFilter EnvironmentFilter to search for mode : either segMode (for finding segment instances) or sylMode (for finding syllable instances) Returns ------- list List of Environments that fit the EnvironmentFilter """ envs = [] if mode == 'segMode': if not isinstance(environment, EnvironmentFilter): return None if all(m not in self for m in environment._middle): return None num_segs = len(environment) possibles = zip(*[self.with_word_boundaries()[i:] for i in range(num_segs)]) lhs_num = environment.lhs_count() middle_num = lhs_num rhs_num = middle_num + 1 for i, p in enumerate(possibles): if p in environment: lhs = p[:lhs_num] middle = p[middle_num] rhs = p[rhs_num:] envs.append(Environment(middle, i + middle_num, lhs, rhs)) lhsZeroes, rhsZeroes = environment.zeroPositions if lhsZeroes: word = [seg for pos,seg in enumerate(self.with_word_boundaries()) if pos not in lhsZeroes] possibles = zip(*[word[i:] for i in range(num_segs)]) for i, p in enumerate(possibles): if environment.without_zeroes_contains(p): lhs = p[:lhs_num] middle = p[middle_num] rhs = p[rhs_num:] envs.append(Environment(middle, i + middle_num, lhs, rhs)) if rhsZeroes: rhsZeroes = [rz+middle_num+1 for rz in rhsZeroes] word = [seg for pos, seg in enumerate(self.with_word_boundaries()) if pos not in rhsZeroes] possibles = zip(*[word[i:] for i in range(num_segs)]) for i, p in enumerate(possibles): if environment.without_zeroes_contains(p): lhs = p[:lhs_num] middle = p[middle_num] rhs = p[rhs_num:] envs.append(Environment(middle, i + middle_num, lhs, rhs)) else: # mode == 'sylMode' reg_exp = environment.generate_regular_expression() word_str = self.with_syllable_and_word_boundaries() for match in re.finditer(reg_exp, word_str, overlapped=True): start = match.start() group_dict = match.groupdict() lhs = group_dict['LHS'] middle = group_dict['MID'] rhs = group_dict['RHS'] envs.append(SyllableEnvironment(start, middle, lhs=lhs, rhs=rhs)) if not envs: return None return envs def find_nonmatch(self, environment, is_sets=False): """ Find all instances of an EnvironmentFilter in the Transcription that match in the middle segments, but don't match on the sides Parameters ---------- environment : EnvironmentFilter EnvironmentFilter to search for Returns ------- list List of Environments that fit the EnvironmentFilter's middle but not the sides """ if not isinstance(environment, EnvironmentFilter): return None if is_sets: #I'm sure this entire block can be reduced to a list comprehension plus any() or all(), but I just #can't figure it out right now. Something like: #if not any([[m in self for m in mid] for mid in environment.middle]): return None found = False for mid in environment.middle: for m in mid: if m in self: found = True break if not found: return None else: if all(m not in self for m in environment.middle): return None num_segs = len(environment) possibles = zip(*[self.with_word_boundaries()[i:] for i in range(num_segs)]) envs = [] lhs_num = environment.lhs_count() middle_num = lhs_num rhs_num = middle_num + 1 for i, p in enumerate(possibles): has_segs = False if is_sets: for mid in environment.middle: if p[middle_num] in mid: has_segs = True else: has_segs = p[middle_num] in environment.middle if p not in environment and has_segs: lhs = p[:lhs_num] middle = p[middle_num] rhs = p[rhs_num:] envs.append(Environment(middle, i + middle_num, lhs, rhs)) if not envs: return None return envs def __contains__(self, other): if isinstance(other, Segment): if other.symbol in self._list: return True elif isinstance(other, str): if other in self._list: return True return False def __setstate__(self, state): if 'stress_pattern' not in state: state['stress_pattern'] = {} if 'boundaries' not in state: state['boundaries'] = {} self.__dict__.update(state) def __hash__(self): return hash(str(self)) def __getitem__(self, key): if isinstance(key,int) or isinstance(key,slice): return self._list[key] raise(KeyError) def __repr__(self): return self.__str__() def __str__(self): temp_list = [] for i,s in enumerate(self._list): if self.stress_pattern and i in self.stress_pattern: s += self.stress_pattern[i] if 'tone' in self.boundaries and i in self.boundaries['tone']: s += self.boundaries['tone'][i] temp_list.append(s) if 'morpheme' in self.boundaries: beg = 0 bound_list = [] for i in self.boundaries['morpheme']: bound_list.append('.'.join(temp_list[beg:i])) bound_list.append('.'.join(temp_list[i:])) return '-'.join(bound_list) else: return '.'.join(temp_list) def __iter__(self): for s in self._list: yield s def __add__(self, other): """ Allow for Transcriptions to be added to get all the segments in each """ if not isinstance(other,Transcription): raise(TypeError) return self._list + other._list def __eq__(self, other): if isinstance(other,list): if len(other) != len(self): return False for i,s in enumerate(self): if s != other[i]: return False return True if not isinstance(other, Transcription): return False if self._list != other._list: return False if self.stress_pattern != other.stress_pattern: return False if self.boundaries != other.boundaries: return False return True def __lt__(self,other): if isinstance(other, Transcription): return self._list < other._list else: return self._list < other def __le__(self,other): if isinstance(other, Transcription): return (self._list == other._list or self._list < other._list) else: return self._list <= other def __ge__(self,other): if isinstance(other, Transcription): return (self._list == other._list or self._list > other._list) else: return self._list >= other def __gt__(self,other): if isinstance(other, Transcription): return self._list > other._list else: return self._list > other def match_segments(self, segments): """ Returns a matching segments from a list of segments Parameters ---------- segments : list List of Segments or strings to filter the Transcription Returns ------- list List of segments (in their original order) that match the segment parameter """ match = [] for s in self: if s in segments: match.append(s) return match def __ne__(self, other): return not self.__eq__(other) def __len__(self): return len(self._list) @property def list(self): return self._list
[docs]class FeatureMatrix(object): """ An object that stores feature values for segments Parameters ---------- name : str Name to give the FeatureMatrix feature_entries : list List of dict with one dictionary per segment, requires the key of symbol which identifies the segment Attributes ---------- name : str An informative identifier for the feature matrix features : list Sorted list of feature names possible_values : set Set of values used in the FeatureMatrix default_value : str Default feature value, usually corresponding to unspecified _features stresses : dict Mapping of stress values to segments that bear that stress places : dict Mapping from place of articulation labels to a feature specification manners : dict Mapping from manner of articulation labels to a feature specification height : dict Mapping from vowel height labels to a feature specification backness : dict Mapping from vowel backness labels to a feature specification vowel_features : str Feature value (i.e., '+voc') that separates vowels from consonants voice_feature : str Feature value (i.e., '+voice') that codes voiced obstruents diph_feature : str Feature value (i.e., '+diphthong' or '.high') that separates diphthongs from monophthongs rounded_feature : str Feature value (i.e., '+round') that codes rounded vowels """ attributes = ['name', '_features', 'vowel_features', 'cons_features', 'voice_features', 'rounded_feature', 'diph_feature', 'possible_values', 'matrix', '_default_value'] def __init__(self, name, feature_entries): self.name = name self._features = None self.vowel_feature = None self.cons_features = None self.voice_features = None self.rounded_feature = None self.diph_feature = None self.possible_values = set() self.matrix = {} self._default_value = 'n' if isinstance(feature_entries, FeatureMatrix): for attr in self.attributes: if hasattr(feature_entries, attr): setattr(self, attr, getattr(feature_entries, attr)) else: for s in feature_entries: self.matrix[s['symbol']] = {k:v for k,v in s.items() if k != 'symbol'} self.possible_values.update({v for k,v in s.items() if k != 'symbol'}) #This if-block never seems to be called, and this whole class' code #appears to treat ._features as a list, so why it should default to #dictionary here is unclear. However, changing to a list causes the #"make feature system from text file" option to break, so better to #leave this alone if self._features is None: self._features = {k for k in s.keys() if k != 'symbol'} def __eq__(self, other): if not isinstance(other, FeatureMatrix): return False if self.matrix == other.matrix: return True return False def default_fill(self, seg_list): for seg in seg_list: self.matrix[seg] = {feature: self.default_value for feature in self._features} @property def trans_name(self): return self.name.split('2')[0] @property def feature_name(self): return self.name.split('2')[-1] def features_to_segments(self, feature_description): """ Given a feature description, return the segments in the inventory that match that feature description Feature descriptions should be either lists, such as ['+feature1', '-feature2'] or strings that can be separated into lists by ',', such as '+feature1,-feature2'. Parameters ---------- feature_description : str, list, or dict Feature values that specify the segments, see above for format Returns ------- list of Segments Segments that match the feature description """ segments = list() if isinstance(feature_description, str): feature_description = feature_description.split(',') #otherwise it's probably a list, leave it be for k,v in self.matrix.items(): if self.feature_match(feature_description, v): segments.append(k) return segments def feature_match(self, features_to_match, segment_description): """ :param features_to_match: a list of strings representing features, e.g. ['+voice','-nasal'] :param segment_description: a dictionary of {feature_name:feature_value} :return: True if features match up with the segment description, e.g. if {'voice':'+', 'nasal':'-'} is in the segment dictionary. Otherwise returns False. """ for feature in features_to_match: if segment_description[feature[1:]] == feature[0]: continue else: return False else: return True def __setstate__(self,state): if 'features' not in state: state['features'] = state['_features'] self.__dict__.update(state) #Backwards compatability if '_default_value' not in state: self._default_value = 'n' if 'places' not in state: self.places = collections.OrderedDict() self.manners = collections.OrderedDict() self.backness = collections.OrderedDict() self.height = collections.OrderedDict() #self.generate_generic_names() if 'cons_column_data' not in state: self.cons_columns = {} self.cons_rows = {} self.vow_columns = {} self.vow_rows = {} self.vowel_feature = None self.voice_feature = None self.rounded_feature = None def __iter__(self): for k in sorted(self.matrix.keys()): yield self.matrix[k] def validate(self): """ Make sure that all segments in the matrix have all the features. If not, add an unspecified value for that feature to them. """ for k,v in self.matrix.items(): for f in self._features: if f not in v: self.matrix[k][f] = self._default_value def set_major_class_features(self, source): self.vowel_feature = source.vowel_feature self.voice_feature = source.voice_feature self.rounded_feature = source.rounded_feature self.diphthong_feature = source.diph_feature @property def default_value(self): return self._default_value @property def features(self): """ Get a list of _features that are used in this feature system Returns ------- list Sorted list of the names of all _features in the matrix """ return sorted(list(self._features)) def add_segment(self,seg,feat_spec): """ Add a segment with a feature specification to the feature system Attributes ---------- seg : str Segment symbol to add to the feature system feat_spec : dictionary Dictionary with _features as keys and feature values as values """ #Validation for f in feat_spec.keys(): if f not in self._features: raise(AttributeError('The segment \'%s\' has a feature \'%s\' that is not defined for this feature matrix' %(seg,f))) # s = Segment(seg) # s.set_features(feat_spec) # self.matrix[seg] = s._features self.matrix[seg] = feat_spec def add_feature(self,feature, default = None): """ Add a feature to the feature system Attributes ---------- feature : str Name of the feature to add to the feature system default : str, optional If specified, set the value for all segments to this value, otherwise use the FeatureMatrix's ``default_value`` """ #self._features.update({feature}) #see note about dictionary vs. list in the __init__() method self._features.append(feature) self._features.sort() if default is None: self.validate() else: for seg,features in self.matrix.items(): for f in self._features: if f not in features: self.matrix[seg][f] = default def valid_feature_strings(self): """ Get all combinations of ``possible_values`` and ``_features`` Returns ------- list List of valid feature strings """ strings = [] for v in self.possible_values: for f in self._features: strings.append(v+f) return strings def categorize(self, seg): """ Categorize a segment into consonant/vowel, place of articulation, manner of articulation, voicing, vowel height, vowel backness, and vowel rounding. For consonants, the category is of the format: ('Consonant', PLACE, MANNER, VOICING) For vowels, the category is of the format: ('Vowel', HEIGHT, BACKNESS, ROUNDED) Diphthongs are categorized differently: ('Diphthong', 'Vowel') Parameters ---------- seg : Segment Segment to categorize Returns ------- tuple or None Returns categories according to the formats above, if any are unable to be calculated, returns None in those places. Returns None if a category cannot be found. """ if seg == '#': return None seg_features = seg.features if seg.feature_match(self.vowel_feature): category = ['Vowel'] if seg.feature_match(self.diph_feature): category.insert(0,'Diphthong') return category for k,v in self.height.items(): if seg.feature_match(v): category.append(k) break else: category.append(None) for k,v in self.backness.items(): if seg.feature_match(v): category.append(k) break else: category.append(None) if seg.feature_match(self.rounded_feature): category.append('Rounded') else: category.append('Unrounded') else: category = ['Consonant'] for k,v in self.places.items(): if seg.feature_match(v): category.append(k) break else: category.append(None) for k,v in self.manners.items(): if seg.feature_match(v): category.append(k) break else: category.append(None) if seg.feature_match(self.voice_feature): category.append('Voiced') else: category.append('Voiceless') return category @property def segments(self): """ Return a list of segment symbols that are specified in the feature system Returns ------- list List of all the segments with feature specifications """ return list(self.matrix.keys()) def seg_to_feat_line(self, symbol): """ Get a list of feature values for a given segment in the order that features are return in get_feature_list Use for display purposes Attributes ---------- symbol : str Segment symbol to look up Returns ------- list List of feature values for the symbol, as well as the symbol itself """ featline = [symbol] + [self.matrix[symbol][feat] for feat in self._features] return featline def specify(self, seg, assign_defaults = False): if isinstance(seg, Segment): symbol = seg.symbol else: symbol = seg try: features = self.matrix[symbol] except KeyError: if assign_defaults: self.matrix[symbol] = {feature:'n' for feature in self.features} features = self.matrix[symbol] else: raise KeyError(symbol) return features def __getitem__(self,item): if isinstance(item,str): #get full feature list for a given segment return self.matrix[item] if isinstance(item, Segment): return self.matrix[item.symbol] if isinstance(item,tuple): #tuple should be (symbol,feature_name) to get only that feature's value try: return self.matrix[item[0]][item[1]] except KeyError: return self.matrix[item[0]][item[1].lower()] def __delitem__(self,item): del self.matrix[item] def __contains__(self,item): return item in list(self.matrix.keys()) def __setitem__(self,key,value): if isinstance(key, str): self.matrix[key] = value if isinstance(key, Segment): self.matrix[key.symbol] = value def __len__(self): return len(self.matrix)
[docs]class Word(object): """An object representing a word in a corpus Information about the attributes are contained in the Corpus' ``attributes``. Attributes ---------- spelling : str A representation of a word that lacks phonological information. transcription : Transcription A representation of a word that includes phonological information. frequency : float Token frequency in a corpus """ word_attributes = {'_corpus': None, '_transcription': None, '_spelling': None, '_transcription_name': None, '_spelling_name': None, '_freq_name': None, 'alt_transcriptions': list(), 'alt_spellings': list(), '_frequency': 0, 'wordtokens': list(), 'descriptors': list()} _freq_names = ['abs_freq', 'freq_per_mil', 'sfreq', 'lowercase_freq', 'log10_freq', 'freq', 'frequency'] def __init__(self, **kwargs): update = kwargs.pop('update', False) # TODO: Don't know what the following code is doing... if update: self.update(update) return self.initDefaults() self._freq_name = kwargs.pop('_freq_name', None) self._spelling_name = kwargs.pop('_spelling_name', None) for key, value in kwargs.items(): if not all([letter.isupper() for letter in key]): key = key.capitalize() if isinstance(value, Decimal): value = float(value) if isinstance(value, tuple): # this block of code is used when loading a corpus for the first time att, value = value if att.att_type == 'numeric': try: value = locale.atof(value) except (ValueError, TypeError): value = float('nan') if hasattr(att, 'is_freq'): self._freq_name = key setattr(self, 'Frequency', float(value)) setattr(self, '_frequency', value) elif att.att_type == 'tier': value = Transcription(value) # The value argument is a list of BaseAnnotation # _list is a list of segment symbol else: # att.att_type == 'spelling' or att.att_type == 'factor': pass setattr(self, key, value) if att.is_default: if att.att_type == 'tier': setattr(self, '_transcription', value) self._transcription_name = key elif att.att_type == 'spelling': setattr(self, '_spelling', value) self._spelling_name = key else: if att.att_type == 'tier': self.alt_transcriptions.append(key) elif att.att_type == 'spelling': self.alt_spellings.append(key) # the following code is reached when adding a word, not when loading a corpus elif isinstance(value, list): # probably a transcription value = Transcription(value) setattr(self, key, value) setattr(self, '_transcription', value) setattr(self, '_transcription_name', key) elif isinstance(value, Transcription): # definitely a transcription setattr(self, key, value) setattr(self, '_transcription', value) setattr(self, '_transcription_name', key) elif isinstance(value, str): if key == 'Spelling': # it's spelling. leave value as-is even though it is only consist of numeric setattr(self, '_spelling', value) setattr(self, '_spelling_name', key) elif key == self._spelling_name: setattr(self, 'spelling', value) setattr(self, '_spelling', value) elif key == 'Frequency' or key == self._freq_name: setattr(self, 'Frequency', float(value)) setattr(self, '_frequency', self.Frequency) #try: # value = float(value) # if key == 'Frequency': # setattr(self, 'Frequency', value) # setattr(self, '_frequency', self.Frequency) #except ValueError: # #it's spelling, leave value as-is # setattr(self, '_spelling', value) # setattr(self, '_spelling_name', key) setattr(self, key, value) elif isinstance(value, (float, int)): if key == 'Frequency' or key == self._freq_name: setattr(self, 'Frequency', value) setattr(self, '_frequency', self.Frequency) setattr(self, key, value) if key not in self.descriptors: self.descriptors.append(key) if self.spelling is None and self.transcription is None: raise(ValueError('Words must be specified with at least a spelling or a transcription.')) if self.spelling is None and self._spelling_name is None: self.Spelling = ''.join(map(str, self._transcription)) self._spelling = self.Spelling self._spelling_name = 'Spelling' if not 'Spelling' in self.descriptors: self.descriptors.append('Spelling') if not hasattr(self, 'Frequency'): self.descriptors.append('Frequency') self._frequency = 0 self.Frequency = 0 if self._transcription_name is None: for d in self.descriptors: if isinstance(getattr(self,d,None), Transcription): self._transcription_name = d break else: self._transcription = None def initDefaults(self): for attribute, default_value in Word.word_attributes.items(): if isinstance(default_value, list): setattr(self, attribute, [x for x in default_value]) elif isinstance(default_value, dict): setattr(self, attribute, default_value.copy()) else: setattr(self, attribute, default_value) @property def frequency(self): # TODO: figure out what different frequencies are doing... #return max([self.Frequency, self._frequency]) return self.Frequency @frequency.setter def frequency(self, value): self.Frequency = value @frequency.deleter def frequency(self): del self.Frequency @property def transcription(self): if self._transcription_name is not None: value = getattr(self, self._transcription_name) else: value = self._transcription return value @transcription.setter def transcription(self, value): if self._transcription_name is not None: setattr(self, self._transcription_name, value) self._transcription = value @transcription.deleter def transcription(self): del self._transcription @property def spelling(self): try: value = getattr(self, self._spelling_name, self._spelling) except (TypeError, AttributeError): value = None #spelling doesn't exist return value @spelling.setter def spelling(self, value): if self._spelling_name is not None: setattr(self, self._spelling_name, value) self._spelling = value @spelling.deleter def spelling(self): del self._spelling def __copy__(self): return Word(update=self) def update(self, old_word): for attribute, value in old_word.__dict__.items(): if not hasattr(self, attribute): setattr(self, attribute, value) for attribute, default_value in Word.word_attributes.items(): if hasattr(old_word, attribute): setattr(self, attribute, getattr(old_word, attribute)) else: setattr(self, attribute, default_value) if attribute == '_freq_name': if self._freq_name is None: self._freq_name = 'Frequency' if hasattr(old_word, 'wordtokens'): self.wordtokens = list() for wt in old_word.wordtokens: self.wordtokens.append(copy.copy(wt)) self.descriptors.extend([att for att in Word.word_attributes if not att.startswith('_')]) if not self._transcription: try: self._transcription = old_word.transcription except AttributeError: try: self._transcription = old_word._transcription except AttributeError: self._transcription = None self.Transcription = self._transcription self._transcription_name = 'Transcription' self.descriptors.append('Transcription') if not self._spelling: try: self._spelling = old_word.spelling except AttributeError: try: self._spelling = old_word._spelling except AttributeError: self._spelling = None self.Spelling = self._spelling self._spelling_name = 'Spelling' self.descriptors.append('Spelling') try: self.Frequency = old_word.frequency except AttributeError: self.Frequency = old_word.Frequency try: self.descriptors.remove('_frequency') except ValueError: pass try: self.descriptors.remove('frequency') except ValueError: pass self.descriptors.append('Frequency') self.descriptors = list(set(self.descriptors)) def get_len(self, tier_name): return len(getattr(self, tier_name)) def enumerate_symbols(self, tier_name, reversed=False): if reversed: word = [k for k in range(self.get_len(tier_name))] word.reverse() for j in word: yield (j, getattr(self, tier_name)[j]) else: for j in range(self.get_len(tier_name)): yield (j, getattr(self, tier_name)[j]) def __hash__(self): return hash((self._spelling, str(self._transcription))) def __getstate__(self): state = self.__dict__.copy() # state['wordtokens'] = [] # state['_corpus'] = None # for k,v in state.items(): # if (k == 'transcription' or k in self.tiers) and v is not None: # state[k] = [x.symbol for x in v] #Only store string symbols return state def __setstate__(self, state): self._transcription = [] self._spelling = '' self._frequency = 0 if 'wordtokens' not in state: state['wordtokens'] = [] if 'descriptors' not in state: state['descriptors'] = ['_spelling','_transcription', '_frequency'] if '_frequency' not in state['descriptors']: state['descriptors'].append('_frequency') try: tiers = state.pop('tiers') for t in tiers: state['descriptors'].append(t) except KeyError: pass self.__dict__.update(state) def add_abstract_tier(self, tier_name, tier_segments): """ Add an abstract tier to the Word Parameters ---------- tier_name : str Attribute name tier_segments: dict Dictionary with keys of the abstract segments (i.e., 'C' or 'V') and values that are sets of segments """ tier = [] for s in self.transcription: for k,v in tier_segments.items(): if s in v: tier.append(k) break setattr(self,tier_name,''.join(tier)) def add_attribute(self, tier_name, value): """ Add an arbitrary attribute to the Word Parameters ---------- tier_name : str Attribute name value: object Attribute value """ setattr(self, tier_name, value) def add_tier(self, tier_name, tier_segments): """Adds a new tier attribute to the Word Parameters ---------- tier_name : str Name for the new tier tier_segments: list of segments Segments that count for inclusion in the tier """ matching_segs = self.transcription.match_segments(tier_segments) new_tier = Transcription(matching_segs) setattr(self,tier_name,new_tier) for wt in self.wordtokens: matching_segs = wt.transcription.match_segments(tier_segments) new_tier = Transcription(matching_segs) setattr(wt,tier_name,new_tier) def remove_attribute(self, attribute_name): """Deletes a tier attribute from a Word Parameters ---------- attribute_name : str Name of tier attribute to be deleted. Notes ----- If attribute_name is not a valid attribute, this function does nothing. It does not raise an error. """ if attribute_name.startswith('_'): return try: delattr(self, attribute_name) except ValueError: pass #attribute_name does not exist def variants(self, sequence_type = 'transcription'): """ Get variants and frequencies for a Word Parameters ---------- sequence_type : str, optional Tier name to get variants Returns ------- dict Dictionary with keys of Transcriptions and values of their frequencies """ try: return collections.Counter(getattr(x, 'Surface_transcription') for x in self.wordtokens) # Buckeye Corpus except AttributeError: pass return collections.Counter(getattr(x, 'transcription') for x in self.wordtokens) def __repr__(self): return '<Word: \'%s\'>' % self.spelling def __str__(self): return self.spelling def __eq__(self, other): if not isinstance(other,Word): return False if self.spelling != other.spelling: return False if self.transcription != other.transcription: return False return True def __ne__(self, other): return not self.__eq__(other) def __lt__(self, other): return self.spelling < other.spelling def __gt__(self, other): return self.spelling > other.spelling def __le__(self, other): return self.spelling <= other.spelling def __ge__(self, other): return self.spelling >= other.spelling
class SyllableEnvironment(object): def __init__(self, start, middle, lhs, rhs): self._start = start self._middle = middle.replace('.', '').strip('-').split(sep='--') self._lhs = lhs.replace('.', '').strip('-').split(sep='--') self._rhs = rhs.replace('.', '').strip('-').split(sep='--') # object 'raw_syllables' should preserve all syllabic and syllable-internal delimiters # I.e., '-' for syllable boundary, '.' for separating onset, neucleus, and coda. # e.g., 'tataim' should be '-.t..a...--.t..a...--...i..m.-'. self.raw_syllables = {'middle':middle, 'lhs': lhs, 'rhs': rhs} @property def middle(self): return self._middle @middle.setter def middle(self, new): self._middle = new @property def lhs(self): return self._lhs @property def rhs(self): return self._rhs @lhs.setter def lhs(self, new): self._lhs = new @rhs.setter def rhs(self, new): self._rhs = new @property def start(self): return self._start @start.setter def start(self, new): self._start = new def __str__(self): lhs = '-'.join(self._lhs) rhs = '-'.join(self._rhs) return lhs + '_' + rhs def print_syl_structure(self, env=True): """ Take the syllable (list of dicts, cf. 'parse_syl' function) and convert it as a printable linear string. Returns ------- str List of dicts. each dict in the list represents one syllable. The keys are 'tone', 'stress', 'onset', 'nucleus', and 'coda' which have a str value env Bool. If True, parse the raw syllable as an env. If False, parse it as the target of the PhonoSearch """ # this one should be called when printing out the syllable structure, e.g., the environment slot of PS result. if env: lhs_output = [] rhs_output = [] for s in self.parse_syl(self.raw_syllables['lhs']): syl_output = [] if "nonseg" in s: syl_str = s['nonseg'] else: syl_output.append("O:{%s}" % s['onset']) if s['onset'] != '' else None syl_output.append("N:{%s}" % s['nucleus']) if s['nucleus'] != '' else None syl_output.append("C:{%s}" % s['coda']) if s['coda'] != '' else None syl_output.append("Tone:{%s}" % s['tone']) if s['tone'] != '' else None syl_output.append("Stress:{%s}" % s['stress']) if s['stress'] != '' else None syl_str = ', '.join(syl_output) syl_str = 'S:{' + syl_str + '}' lhs_output.append(syl_str) for s in self.parse_syl(self.raw_syllables['rhs']): syl_output = [] if "nonseg" in s: syl_str = s['nonseg'] else: syl_output.append("O:{%s}" % s['onset']) if s['onset'] != '' else None syl_output.append("N:{%s}" % s['nucleus']) if s['nucleus'] != '' else None syl_output.append("C:{%s}" % s['coda']) if s['coda'] != '' else None syl_output.append("Tone:{%s}" % s['tone']) if s['tone'] != '' else None syl_output.append("Stress:{%s}" % s['stress']) if s['stress'] != '' else None syl_str = ', '.join(syl_output) syl_str = 'S:{' + syl_str + '}' rhs_output.append(syl_str) lhs = ', '.join(lhs_output) + ' ' rhs = ' ' + ', '.join(rhs_output) return lhs + '_' + rhs else: mid_output = [] for s in self.parse_syl(self.raw_syllables['middle']): syl_output = [] if "nonseg" in s: syl_str = s['nonseg'] else: syl_output.append("O:{%s}" % s['onset']) if s['onset'] != '' else None syl_output.append("N:{%s}" % s['nucleus']) if s['nucleus'] != '' else None syl_output.append("C:{%s}" % s['coda']) if s['coda'] != '' else None syl_output.append("Tone:{%s}" % s['tone']) if s['tone'] != '' else None syl_output.append("Stress:{%s}" % s['stress']) if s['stress'] != '' else None syl_str = ', '.join(syl_output) syl_str = 'S:{' + syl_str + '}' mid_output.append(syl_str) return ', '.join(mid_output) def __repr__(self): return self.__str__() def __hash__(self): return hash((self._start, self._middle, self._lhs, self._rhs)) def __eq__(self, other): """ Two Environments are equal if they share a left AND right hand side An empty lhs or rhs is an automatic match """ if not isinstance(other, SyllableEnvironment): return False if other.lhs and other.lhs != self._lhs: return False if other.rhs and other.rhs != self._rhs: return False if other.start != self._start: return False return True def __ne__(self, other): return not self.__eq__(other) def parse_syl(self, syl_rep): """ Parse linear representation of syllable into list of dicts Parameters ---------- syl_rep : str Linearly represented syllables. for example, the syllable information for the word 'ma3shang4' (two syllables) is linearlly represented as '-.m..a...3--.ʂ..a..ŋ.4-' where a pair of '-'s demarcates a syllable, and a pair of '.'s demarcates a syllable-internal unit such as onset nucleus and coda. i.e., a syllable is '-(stress).(onset)..(nucleus)..(coda).(tone)-' Returns ------- list List of dicts. each dict in the list represents one syllable. The keys are 'tone', 'stress', 'onset', 'nucleus', and 'coda' which have a str value """ syllables = syl_rep.split(sep='--') syl_output = [] for syl in syllables: syl = syl.strip('-') tone_str_onc = [] if len(syl) == 1: syl_output.append({'nonseg': syl}) continue elif len(syl) == 0: continue if syl[-1] != '.': # if tone exists tone_str_onc.append(syl[-1]) syl = syl[:-1] else: tone_str_onc.append('') if syl[0] != '.': # if stress exists tone_str_onc.append(syl[0]) syl = syl[1:] else: tone_str_onc.append('') tone_str_onc.extend([i.strip('.') for i in re.findall('\..*?\.', syl)]) syl_dict = dict(zip(['tone', 'stress', 'onset', 'nucleus', 'coda'], tone_str_onc)) syl_output.append(syl_dict) return syl_output
[docs]class Environment(object): """ Specific sequence of segments that was a match for an EnvironmentFilter Parameters ---------- middle : str Middle segment position : int Position of the middle segment in the word (to differentiate between repetitions of an environment in the same word lhs : list, optional Segments to the left of the middle segment rhs : list, optional Segments to the right of the middle segment """ def __init__(self, middle, position, lhs=None, rhs=None): self.middle = middle self.position = position self.lhs = lhs self.rhs = rhs self.lhs_string = None self.rhs_string = None self.middle_string = None def __getitem__(self, key): if self.lhs is not None: if key < len(self.lhs): return self.lhs[key] elif key == len(self.lhs): return self.middle elif self.rhs is not None: return self.rhs[key - len(self.lhs) - 1] else: raise(KeyError('Index out of bounds')) else: if key == 0: return self.middle elif self.rhs is not None: return self.rhs[key - 1] else: raise(KeyError('Index out of bounds')) def __str__(self): elements = [] if self.lhs_string is not None: elements.append(self.lhs_string) elif self.lhs is not None: elements.append(''.join(self.lhs)) else: elements.append('') if self.rhs_string is not None: elements.append(self.rhs_string) elif self.rhs is not None: elements.append(''.join(self.rhs)) else: elements.append('') return '_'.join(elements) def __repr__(self): return self.__str__() def __hash__(self): return hash((self.lhs, self.position, self.middle, self.rhs)) def __eq__(self,other): """ Two Environments are equal if they share a left AND right hand side An empty lhs or rhs is an automatic match """ if not isinstance(other,Environment): return False if other.lhs and other.lhs != self.lhs: return False if other.rhs and other.rhs != self.rhs: return False if other.position != self.position: return False return True def __ne__(self,other): return not self.__eq__(other)
class SyllableEnvironmentFilter(object): def __init__(self, inventory, middle_syllables, lhs=list(), rhs=list()): self._middle_syllables = middle_syllables self._lhs = lhs self._rhs = rhs self._inventory = inventory @property def middle(self): return self._middle_syllables @property def lhs(self): return self._lhs @property def rhs(self): return self._rhs @property def inventory(self): return self._inventory @inventory.setter def inventory(self, new): self._inventory = new @middle.setter def middle(self, new): self._middle_syllables = new @lhs.setter def lhs(self, new): self._lhs = new @rhs.setter def rhs(self, new): self._rhs = new def generate_nonsegs_re(self, syllable): re_group = set() for symbol in syllable['nonsegs']: if symbol in SPECIAL_SYMBOL_RE: symbol = '\\' + symbol re_group.add(symbol) else: re_group.add(symbol) nonsegs_re = '(?:-(?#NONSEGS)' + '|'.join(re_group) + '-)' return nonsegs_re def generate_constituent_re(self, syllable, constituent): constituent_re = list() for unit in syllable[constituent]['contents']: re_group = set() for seg in unit: if seg in SPECIAL_SYMBOL_RE: seg = '\\' + seg re_group.add(seg) else: re_group.add(seg) unit_re = '(?:' + '|'.join(re_group) + ')' constituent_re.append(unit_re) if syllable[constituent]['search_type'] == 'Exactly matches': constituent_re = '(?:\.(?#' + constituent.upper() + ')' + ''.join(constituent_re) + '\.)' else: # Not Exactly matches general_unit_re_group = set() segs = list(self.inventory.segs.keys()) segs.remove('#') # Because # is a boundary symbol for seg in segs: if seg in SPECIAL_SYMBOL_RE: seg = '\\' + seg general_unit_re_group.add(seg) else: general_unit_re_group.add(seg) general_unit_re = '(?:(?#ALLSEGS)' + '|'.join(general_unit_re_group) + ')*' if syllable[constituent]['search_type'] == 'Starts with': constituent_re = '(?:\.(?#' + constituent.upper() + ')' + ''.join(constituent_re) + general_unit_re + '\.)' elif syllable[constituent]['search_type'] == 'Ends with': constituent_re = '(?:\.(?#' + constituent.upper() + ')' + general_unit_re + ''.join(constituent_re) + '\.)' else: # Minimally contains constituent_re = '(?:\.(?#' + constituent.upper() + ')' + general_unit_re + general_unit_re.join(constituent_re) + general_unit_re + '\.)' return constituent_re def generate_stress_re(self, syllable): re_group = set() for stress in syllable['stress']: if stress == 'None': stress = '' re_group.add(stress) elif stress in SPECIAL_SYMBOL_RE: stress = '\\' + stress re_group.add(stress) else: re_group.add(stress) stress_re = '(?:(?#STRESS)' + '|'.join(re_group) + ')' return stress_re def generate_tone_re(self, syllable): re_group = set() for tone in syllable['tone']: if tone == 'None': tone = '' re_group.add(tone) elif tone in SPECIAL_SYMBOL_RE: tone = '\\' + tone re_group.add(tone) else: re_group.add(tone) tone_re = '(?:(?#TONE)' + '|'.join(re_group) + ')' return tone_re def generate_syllable_re(self, syllable): if syllable['nonsegs']: syllable_re = self.generate_nonsegs_re(syllable) else: stress_re = self.generate_stress_re(syllable) tone_re = self.generate_tone_re(syllable) onset_re = self.generate_constituent_re(syllable, 'onset') nucleus_re = self.generate_constituent_re(syllable, 'nucleus') coda_re = self.generate_constituent_re(syllable, 'coda') syllable_re = '(?:-(?#SYLLABLE)' + stress_re + onset_re + nucleus_re + coda_re + tone_re + '-)' return syllable_re def generate_regular_expression(self): """ This returns the regular expression associated with the environment :return: """ lhs_re = '' for syllable in self._lhs: syllable_re = self.generate_syllable_re(syllable) lhs_re += syllable_re lhs_re = '(?P<LHS>' + lhs_re + ')' mid_re = '' for syllable in self._middle_syllables: syllable_re = self.generate_syllable_re(syllable) mid_re += syllable_re mid_re = '(?P<MID>' + mid_re + ')' rhs_re = '' for syllable in self._rhs: syllable_re = self.generate_syllable_re(syllable) rhs_re += syllable_re rhs_re = '(?P<RHS>' + rhs_re + ')' final_re = lhs_re + mid_re + rhs_re #print(final_re) return re.compile(final_re) def __str__(self): ls = '' for syllable in self._lhs: ls += self.str_syllable(syllable) md = '' for syllable in self._middle_syllables: md += self.str_syllable(syllable) rs = '' for syllable in self._rhs: rs += self.str_syllable(syllable) bound1 = "-" if ls and (md or rs) else "" bound2 = "-" if bound1 and rs else "" return ls + bound1 + md + bound2 + rs def str_syllable(self, syllable): onset = ''.join([self.str_set(set) for set in syllable['onset']['contents']]) nucleus = ''.join([self.str_set(set) for set in syllable['nucleus']['contents']]) coda = ''.join([self.str_set(set) for set in syllable['coda']['contents']]) return onset + nucleus + coda def str_set(self, segments): inv_set = set(self._inventory.segs.keys()) inv_set.remove('#') if segments != inv_set: return "{" + ''.join(segments) + "}" else: return "{*}"
[docs]class EnvironmentFilter(object): """ Filter to use for searching words to generate Environments that match Parameters ---------- middle_segments : set Set of segments to center environments lhs : list, optional List of set of segments on the left of the middle rhs : list, optional List of set of segments on the right of the middle """ def __init__(self, middle_segments, lhs=None, rhs=None, zeroPositions=None): self.original_middle = middle_segments self.special_match_symbol = '*' if lhs is not None: lhs = tuple(lhs) self.lhs = lhs if rhs is not None: rhs = tuple(rhs) self.rhs = rhs self.lhs_string = None self.rhs_string = None self._sanitize() if zeroPositions is None: self.zeroPositions = (None, None) else: self.zeroPositions = zeroPositions @property def middle(self): return self.original_middle @middle.setter def middle(self, middle_segments): self.original_middle = middle_segments self._sanitize() def _sanitize(self): if self.lhs is not None: new_lhs = [] for seg_set in self.lhs: if not isinstance(seg_set,frozenset): new_lhs.append(frozenset(seg_set)) else: new_lhs.append(seg_set) self.lhs = tuple(new_lhs) if self.rhs is not None: new_rhs = [] for seg_set in self.rhs: if not isinstance(seg_set,frozenset): new_rhs.append(frozenset(seg_set)) else: new_rhs.append(seg_set) self.rhs = tuple(new_rhs) if not isinstance(self.middle, frozenset): self.middle = frozenset(self.middle) self._middle = set() for m in self.middle: if isinstance(m, str): self._middle.add(m) elif isinstance(m, (list, tuple, set)): self._middle.update(m) def is_applicable(self, sequence): """ Check whether the Environment filter is applicable to the sequence (i.e., the sequence must be greater or equal in length to the EnvironmentFilter) Parameters ---------- sequence : list Sequence to check applicability Returns ------- bool True if the sequence is equal length or longer than the EnvironmentFilter """ if len(sequence) < len(self): return False return True def compile_re_pattern(self): pass def lhs_count(self): """ Get the number of elements on the left hand side Returns ------- int Length of the left hand side """ if self.lhs is None: return 0 return len(self.lhs) def rhs_count(self): """ Get the number of elements on the right hand side Returns ------- int Length of the right hand side """ if self.rhs is None: return 0 return len(self.rhs) def set_lhs(self, lhs): self.lhs = lhs self.compile_re_pattern() def set_rhs(self, rhs): self.rhs = rhs self.compile_re_pattern() def __iter__(self): if self.lhs is not None: for s in self.lhs: yield s yield self._middle if self.rhs is not None: for s in self.rhs: yield s def __len__(self): length = 1 if self.lhs is not None: length += len(self.lhs) if self.rhs is not None: length += len(self.rhs) return length def __str__(self): elements = [] if self.lhs_string is not None: elements.append(self.lhs_string) elif self.lhs is not None: elements.append(''.join('{' + ','.join(x) + '}' for x in self.lhs)) else: elements.append('') if self.rhs_string is not None: elements.append(self.rhs_string) elif self.rhs is not None: elements.append(''.join('{' + ','.join(x) + '}' for x in self.rhs)) else: elements.append('') return '_'.join(elements) def __eq__(self, other): if not hasattr(other,'lhs'): return False if not hasattr(other,'rhs'): return False if self.lhs != other.lhs: return False if self.rhs != other.rhs: return False return True def __hash__(self): return hash((self.rhs, self.lhs)) def __contains__(self, sequence): for i, s in enumerate(self): if self.special_match_symbol in s: continue if sequence[i] not in s: return False return True def without_zeroes_contains(self, sequence): for i, s in enumerate(self.without_zero_positions()): if self.special_match_symbol in s: continue if sequence[i] not in s: return False return True def without_zero_positions(self): e = list() if self.lhs is not None: for i,s in enumerate(self.lhs): if i in self.zeroPositions[0]: continue else: e.append(s) e.append(self._middle) if self.rhs is not None: for i,s in enumerate(self.rhs): if i in self.zeroPositions[1]: continue else: e.append(s) return e def generate_regular_expression(self): RegExp = '' for set in self.lhs: RegExp += '(%s)' % '|'.join(set) RegExp += '(?P<MID>%s)' % '|'.join(self.middle) for set in self.rhs: RegExp += '(%s)' % '|'.join(set) return RegExp
[docs]class Attribute(object): """ Attributes are for collecting summary information about attributes of Words or WordTokens, with different types of attributes allowing for different behaviour Parameters ---------- name : str Python-safe name for using `getattr` and `setattr` on Words and WordTokens att_type : str Either 'spelling', 'tier', 'numeric' or 'factor' display_name : str Human-readable name of the Attribute, defaults to None default_value : object Default value for initializing the attribute Attributes ---------- name : string Python-readable name for the Attribute on Word and WordToken objects display_name : string Human-readable name for the Attribute default_value : object Default value for the Attribute. The type of `default_value` is dependent on the attribute type. Numeric Attributes have a float default value. Factor and Spelling Attributes have a string default value. Tier Attributes have a Transcription default value. range : object Range of the Attribute, type depends on the attribute type. Numeric Attributes have a tuple of floats for the range for the minimum and maximum. The range for Factor Attributes is a set of all factor levels. The range for Tier Attributes is the set of segments in that tier across the corpus. The range for Spelling Attributes is None. """ ATT_TYPES = ['spelling', 'tier', 'numeric', 'factor'] def __init__(self, name, att_type, display_name = None, default_value = None, is_default = False): self.name = name self.att_type = att_type self._display_name = display_name self.is_default = is_default # Special treatment for the 'frequency' attribute type selected when creating corpus from a file. # freq should basically work like other numerics in terms of summary presentation so att_type = 'numeric' # but should mark it with self.is_freq if self.att_type == 'freq': self.att_type = 'numeric' self.is_freq = True if self.att_type == 'numeric': self._range = [0,0] if default_value is not None and isinstance(default_value,(int,float)): self._default_value = default_value else: self._default_value = 0 elif self.att_type == 'factor': if default_value is not None and isinstance(default_value,str): self._default_value = default_value else: self._default_value = '' if default_value: self._range = set([default_value]) else: self._range = set() elif self.att_type == 'spelling': self._range = None if default_value is not None and isinstance(default_value,str): self._default_value = default_value else: self._default_value = '' elif self.att_type == 'tier': self._range = set() self._delim = None if default_value is not None and isinstance(default_value,Transcription): self._default_value = default_value else: self._default_value = Transcription(None) def __gt__(self, other): return self.display_name < other.display_name def __lt__(self, other): return not self.__gt__(other) @property def delimiter(self): if self.att_type != 'tier': return None else: return self._delim @delimiter.setter def delimiter(self, value): self._delim = value @staticmethod def guess_type(values, trans_delimiters = None): """ Guess the attribute type for a sequence of values Parameters ---------- values : list List of strings to evaluate for the attribute type trans_delimiters : list, optional List of delimiters to look for in transcriptions, defaults to ``.``, ``;``, and ``,`` Returns ------- str Attribute type that had the most success in parsing the values specified """ if trans_delimiters is None: trans_delimiters = ['.',' ', ';', ','] probable_values = {x: 0 for x in Attribute.ATT_TYPES} for i,v in enumerate(values): try: t = float(v) probable_values['numeric'] += 1 continue except ValueError: for d in trans_delimiters: if d in v: probable_values['tier'] += 1 break else: if v in [v2 for j,v2 in enumerate(values) if i != j]: probable_values['factor'] += 1 else: probable_values['spelling'] += 1 return max(probable_values.items(), key=operator.itemgetter(1))[0] @staticmethod def sanitize_name(name): """ Sanitize a display name into a Python-readable attribute name Parameters ---------- name : string Display name to sanitize Returns ------- str Sanitized name """ name = re.sub(r'\W', '', name) #.lower()) name = name if all([s.isupper() for s in name]) else name.capitalize() return name def __hash__(self): return hash(self.name) def __repr__(self): return '<Attribute of type {} with name \'{}\'>'.format(self.att_type, self.name) def __str__(self): return self.display_name def __eq__(self,other): if isinstance(other,Attribute): if self.name == other.name: return True if isinstance(other,str): if self.name == other: return True return False @property def display_name(self): if self._display_name is not None: return self._display_name#.lower() return self.name @property def default_value(self): return self._default_value @default_value.setter def default_value(self, value): self._default_value = value self._range = set([value]) @property def range(self): return self._range def update_range(self,value): """ Update the range of the Attribute with the value specified. If the attribute is a Factor, the value is added to the set of levels. If the attribute is Numeric, the value expands the minimum and maximum values, if applicable. If the attribute is a Tier, the value (a segment) is added to the set of segments allowed. If the attribute is Spelling, nothing is done. Parameters ---------- value : object Value to update range with, the type depends on the attribute type """ if value is None: return if self.att_type == 'numeric': if isinstance(value, str): try: value = float(value) except ValueError: self.att_type = 'spelling' self._range = None return if value < self._range[0]: self._range[0] = value elif value > self._range[1]: self._range[1] = value elif self.att_type == 'factor': self._range.add(value) #if len(self._range) > 1000: # self.att_type = 'spelling' # self._range = None elif self.att_type == 'tier': if isinstance(self._range, list): self._range = set(self._range) self._range.update([x for x in value])
[docs]class Inventory(object): """ Inventories contain information about a Corpus' segmental inventory. This class exists mainly for the purposes of saving and loading user data. When a user loads a corpus into PCT, the Inventory is passed to the constructor of an InventoryModel (corpustools\gui\models). The InventoryModel is what the user will interact with, and it has several useful functions for analyzing, modifying and sorting the contents of an inventory. The InventoryModel is also connected with an InventoryView (corpustools\gui\views) for the purposes of displaying inventory charts in PCT. User data is saved using Python's built-in pickle module. However, Qt objects (like the InventoryModel) cannot be pickled. Instead, the data from the model is copied to this Inventory class, which is a native Python object and can be properly pickled. Parameters ---------- data : dict, optional Mapping from segment symbol to Segment objects Attributes ---------- _features : list List of all _features used as specifications for segments possible_values : set Set of values that segments use for _features stresses : dict Mapping of stress values to segments that bear that stress places : dict Mapping from place of articulation labels to sets of segments manners : dict Mapping from manner of articulation labels to sets of segments height : dict Mapping from vowel height labels to sets of segments backness : dict Mapping from vowel backness labels to sets of segments vowel_features : str Feature value (i.e., '+voc') that separates vowels from consonants voice_feature : str Feature value (i.e., '+voice') that codes voiced obstruents diph_feature : str Feature value (i.e., '+diphthong' or '.high') that separates diphthongs from monophthongs rounded_feature : str Feature value (i.e., '+round') that codes rounded vowels """ inventory_attributes = {'_data': list(), 'segs': {'#': Segment('#')}, 'syllables': dict(), 'features': list(), 'possible_values': list(), 'stresses': list(), 'stress_types': dict(), 'tone_types': dict(), 'consColumns': set(), 'vowelColumns': set(), 'vowelRows': set(), 'consRows': set(), 'cons_column_data': {}, 'cons_row_data': {}, 'vowel_column_data': {}, 'vowel_row_data': {}, 'uncategorized': list(), 'all_rows': dict(), 'all_columns': dict(), 'vowel_column_offset': int(), 'vowel_row_offset': int(), 'cons_column_header_order': dict(), 'cons_row_header_order': dict(), 'vowel_row_header_order': dict(), 'vowel_column_header_order': dict(), 'consList': list(), 'vowelList': list(), 'non_segment_symbols': ['#', '-', '='], 'vowel_features': [None], 'cons_features': [None], 'voice_feature': None, 'rounded_feature': None, 'diph_feature': None, 'isNew': True, 'filterNames': False, 'minimum_features': {'hayes' : ['consonantal', 'labial', 'coronal', 'labiodental', 'anterior', 'dorsal', 'back', 'sonorant', 'delayed_release', 'nasal', 'continuant', 'trill', 'tap', 'lateral', 'front', 'back', 'high', 'low'], 'spe' : ['voc', 'ant', 'cor', 'high', 'low', 'back', 'son', 'lat', 'nasal']} } def __init__(self, update=False): if update: self.update(update) self.isNew = False return self.initDefaults() def initDefaults(self): for attribute, default_value in Inventory.inventory_attributes.items(): if isinstance(default_value, list): setattr(self, attribute, [x for x in default_value]) elif isinstance(default_value, dict): setattr(self, attribute, default_value.copy()) else: setattr(self, attribute, default_value) def update(self, source): for attribute, default_value in Inventory.inventory_attributes.items(): if hasattr(source, attribute): setattr(self, attribute, getattr(source, attribute)) else: if isinstance(default_value, list): setattr(self, attribute, [x for x in default_value]) elif isinstance(default_value, dict): setattr(self, attribute, default_value.copy()) else: setattr(self, attribute, default_value) def __getstate__(self): state = self.__dict__.copy() return state def __setstate__(self, state): if 'stresses' not in state: state['stresses'] = collections.OrderedDict() if 'places' not in state: state['places'] = collections.OrderedDict() if 'manners' not in state: state['manners'] = collections.OrderedDict() if 'height' not in state: state['height'] = collections.OrderedDict() if 'backness' not in state: state['backness'] = collections.OrderedDict() if 'vowel_features' not in state: state['vowel_features'] = None if 'voice_feature' not in state: state['voice_feature'] = None if 'diph_feature' not in state: state['diph_feature'] = None if 'rounded_feature' not in state: state['rounded_feature'] = None self.__dict__.update(state) def save(self, model): """ Takes an InventoryModel as input, and updates attributes. This is called in main.py in saveCorpus(). """ for attribute in Inventory.inventory_attributes: if hasattr(model, attribute): setattr(self, attribute, getattr(model, attribute)) # else: # pass def __len__(self): return len(self.segs.keys()) def keys(self): return self.segs.keys() def values(self): return self.segs.values() def items(self): return self.segs.items() def __getitem__(self, key): if isinstance(key, slice): return sorted(self.segs.keys())[key] if isinstance(key, str): return self.segs[key] if isinstance(key, Segment): return self.segs[key.symbol] def __setitem__(self, key, value): self.segs[key] = value def __iter__(self): for k in sorted(self.segs.keys()): if k in self.non_segment_symbols: continue yield self.segs[k] def __contains__(self, item): if isinstance(item, str): return item in self.segs.keys() elif isinstance(item, Segment): return item.symbol in self.segs.keys() return False def valid_feature_strings(self): """ Get all combinations of ``possible_values`` and ``_features`` Returns ------- list List of valid feature strings """ strings = [] for v in self.possible_values: for f in self.features: strings.append(v+f) return strings def features_to_segments(self, feature_description): """ Given a feature description, return the segments in the inventory that match that feature description Feature descriptions should be either lists, such as ['+feature1', '-feature2'] or strings that can be separated into lists by ',', such as '+feature1,-feature2'. Parameters ---------- feature_description : string or list Feature values that specify the segments, see above for format Returns ------- list of Segments Segments that match the feature description """ segments = [] if isinstance(feature_description, str): feature_description = feature_description.split(',') for k,v in self.segs.items(): if v.feature_match(feature_description): segments.append(k) return segments def update_features(self, specifier): for seg in self.segs: if seg in self.non_segment_symbols: continue self.segs[seg].features = specifier.specify(seg, assign_defaults=True) self.cons_features = specifier.cons_features if hasattr(specifier, 'cons_features') else [None] self.vowel_features = specifier.vowel_features if hasattr(specifier, 'vowel_features') else [None] self.voice_feature = specifier.voice_feature if hasattr(specifier, 'voice_feature') else None self.rounded_feature = specifier.rounded_feature if hasattr(specifier, 'rounded_feature') else None self.diphthong_feature = specifier.diph_feature if hasattr(specifier, 'diph_feature') else None self.features = specifier.features self.possible_values = specifier.possible_values
[docs]class Corpus(object): """ Lexicon to store information about Words, such as transcriptions, spellings and frequencies Parameters ---------- name : string Name to identify Corpus Attributes ---------- name : str Name of the corpus, used only for easy of reference attributes : list of Attributes List of Attributes that Words in the Corpus have wordlist : dict Dictionary where every key is a unique string representing a word in a corpus, and each entry is a Word object words : list of strings All the keys for the wordlist of the Corpus specifier : FeatureSpecifier See the FeatureSpecifier object inventory : Inventory Inventory that contains information about segments in the Corpus """ corpus_attributes = {'name':'corpus', 'wordlist': dict(), '_discourse': None, 'specifier': None, 'inventory': None, 'inventoryModel': None, 'has_frequency': True, 'has_spelling':False, 'has_wordtokens':False, 'has_audio': False, 'wav_path': None, '_attributes': list(), '_version': currentPCTversion } basic_attributes = ['spelling','transcription','frequency'] def __init__(self, name, update=False): if update: self.update(update) return self.initDefaults() self.name = name def initDefaults(self): for attribute, default_value in Corpus.corpus_attributes.items(): if attribute == 'inventory': setattr(self, attribute, Inventory()) # elif attribute == '_attributes': # setattr(self, attribute, [Attribute('spelling', 'spelling'), # Attribute('transcription', 'tier'), # Attribute('frequency', 'numeric')]) elif isinstance(default_value, list): setattr(self, attribute, [x for x in default_value]) elif isinstance(default_value, dict): setattr(self, attribute, default_value.copy()) else: setattr(self, attribute, default_value) self._version = currentPCTversion def update(self, old_corpus): for attribute,default_value in Corpus.corpus_attributes.items(): if hasattr(old_corpus, attribute): setattr(self, attribute, getattr(old_corpus, attribute)) else: setattr(self, attribute, default_value) self._version = currentPCTversion def update_wordlist(self, new_wordlist): self.wordlist = dict() for word in new_wordlist: self.add_word(word) @property def has_transcription(self): for a in self.attributes: if a.att_type == 'tier' and len(a.range) > 0: return True w = self.random_word() if w.transcription is not None: return True return False def __eq__(self, other): if not isinstance(other,Corpus): return False if self.wordlist != other.wordlist: return False return True def __iadd__(self, other): for a in other.attributes: if a not in self.attributes: self.add_attribute(a) for w in other: try: sw = self.find(w.spelling) sw.frequency += w.frequency for a in self.attributes: try: if getattr(sw, a.name) == a.default_value and getattr(w, a.name) != a.default_value: setattr(sw, a.name, copy.copy(getattr(w, a.name))) except AttributeError: pass for wt in w.wordtokens: sw.wordtokens.append(copy.copy(wt)) except KeyError: self.add_word(copy.copy(w)) if self.specifier is None and other.specifier is not None: self.set_feature_matrix(other.specifier) self.inventory.segs.update(other.inventory.segs) return self def key(self, word): key = word.spelling if self[key] == word: return key count = 0 while True: count += 1 key = '{} ({})'.format(word.spelling,count) try: if self[key] == word: return key except KeyError: break def keys(self): for k in sorted(self.wordlist.keys()): yield k def set_default_representations(self): for att in self.attributes: if att.att_type == 'tier': if att.is_default: self.default_transcription = att else: self.alternative_transcriptions.append(att) elif att.att_type == 'spelling': if att.is_default: self.default_spelling = att else: self.alternative_spellings.append(att) def generate_alternative_inventories(self): for att in self.alternative_transcriptions: altinv = set() for word in self: transcription = getattr(word, att.name) for x in transcription: altinv.add(x) self.alternative_inventories[att.name] = Inventory() for seg in altinv: #for some reason, this loop alters the contents of self.inventory, which it shouldn't self.alternative_inventories[att.name].segs[seg] = Segment(seg, self.specifier.specify(seg, assign_defaults=True)) @property def all_inventories(self): inventories = dict() if hasattr(self, 'lexicon'): inventories[self.default_transcription.display_name] = self.lexicon.inventory else: inventories[self.default_transcription.display_name] = self.inventory for name,inv in self.alternative_inventories.items(): inventories[name] = inv return inventories.items() def retranscribe(self, segmap): self.inventory = Inventory() for word in self.wordlist: T = Transcription([segmap[seg] for seg in self.wordlist[word].transcription]) self.wordlist[word].transcription = T #self.update_inventory(self.wordlist[word].transcription) for seg in T: if seg not in self.inventory: self.inventory.segs[seg] = Segment(seg) self.inventory.segs[seg].features = self.specifier[seg] def subset(self, filters, mode): """ Generate a subset of the corpus based on filters. Filters for Numeric Attributes should be tuples of an Attribute (of the Corpus), a comparison callable (``__eq__``, ``__neq__``, ``__gt__``, ``__gte__``, ``__lt__``, or ``__lte__``) and a value to compare all such attributes in the Corpus to. Filters for Factor Attributes should be tuples of an Attribute, and a set of levels for inclusion in the subset. Other attribute types cannot currently be the basis for filters. Parameters ---------- filters : list of tuples See above for format mode : str How multiple filters are processed. Either 'andMode' or 'orMode'. Returns ------- Corpus Subset of the corpus that matches the filter conditions """ new_corpus = Corpus('') new_corpus._attributes = [Attribute(x.name, x.att_type, x.display_name) for x in self.attributes] new_corpus.inventory = self.inventory if mode == 'andMode': for word in self: for f in filters: if f[0].att_type == 'numeric': op = f[1] if not op(getattr(word, f[0].name), f[2]): break elif f[0].att_type == 'factor': if getattr(word, f[0].name) not in f[1]: break else: new_corpus.add_word(word) else: for word in self: for f in filters: if f[0].att_type == 'numeric': op = f[1] if op(getattr(word, f[0].name), f[2]): new_corpus.add_word(word) break elif f[0].att_type == 'factor': if getattr(word, f[0].name) in f[1]: new_corpus.add_word(word) break return new_corpus @property def attributes(self): #'transcription', 'spelling' and 'frequency' are special attributes which are acually # methods decorated with @property #these methods return the value of word._transcription, word._spelling, or word._frequency #we don't want to put them into the GUI column headers because that will lead to duplication return sorted([a for a in self._attributes if not a.name in ('spelling', 'transcription', 'frequency')]) @property def words(self): return sorted(list(self.wordlist.keys())) def symbol_to_segment(self, symbol): for seg in self.inventory: if seg.symbol == symbol: return seg else: raise CorpusIntegrityError('Could not find {} in the inventory'.format(symbol)) def features_to_segments(self, feature_description): """ Given a feature description, return the segments in the inventory that match that feature description Feature descriptions should be either lists, such as ['+feature1', '-feature2'] or strings that can be separated into lists by ',', such as '+feature1,-feature2'. Parameters ---------- feature_description : string or list Feature values that specify the segments, see above for format Returns ------- list of Segments Segments that match the feature description """ segments = list() if isinstance(feature_description,str): feature_description = feature_description.split(',') for k,v in self.inventory.items(): if v.feature_match(feature_description): segments.append(k) return segments def segment_to_features(self, seg): """ Given a segment, return the _features for that segment. Parameters ---------- seg : string or Segment Segment or Segment symbol to look up Returns ------- dict Dictionary with keys as _features and values as featue values """ try: features = self.specifier.matrix[seg] except TypeError: features = self.specifier.matrix[seg.symbol] return features def add_abstract_tier(self, attribute, spec): """ Add a abstract tier (currently primarily for generating CV skeletons from tiers). Specifiers for abstract tiers should be dictionaries with keys that are the abstract symbol (such as 'C' or 'V') and the values are iterables of segments that should count as that abstract symbols (such as all consonants or all vowels). Currently only operates on the ``transcription`` of words. Parameters ---------- attribute : Attribute Attribute to add/replace spec : dict Mapping for creating abstract tier """ for i,a in enumerate(self._attributes): if attribute.name == a.name: self._attributes[i] = attribute break else: self._attributes.append(attribute) for word in self: word.add_abstract_tier(attribute.name,spec) attribute.update_range(getattr(word,attribute.name)) def add_attribute(self, attribute, initialize_defaults = False): """ Add an Attribute of any type to the Corpus or replace an existing Attribute. Parameters ---------- attribute : Attribute Attribute to add or replace initialize_defaults : boolean If True, words will have this attribute set to the ``default_value`` of the attribute, defaults to False """ for i,a in enumerate(self._attributes): if attribute.display_name == a.display_name: self._attributes[i] = attribute break else: self._attributes.append(attribute) if initialize_defaults: for word in self: word.add_attribute(attribute.name,attribute.default_value) def add_count_attribute(self, attribute, sequence_type, spec): """ Add an Numeric Attribute that is a count of a segments in a tier that match a given specification. The specification should be either a list of segments or a string of the format '+feature1,-feature2' that specifies the set of segments. Parameters ---------- attribute : Attribute Attribute to add or replace sequence_type : string Specifies whether to use 'spelling', 'transcription' or the name of a transcription tier to use for comparisons spec : list or str Specification of what segments should be counted """ if isinstance(attribute,str): attribute = Attribute(attribute,'numeric') for i,a in enumerate(self._attributes): if attribute.name == a.name: self._attributes[i] = attribute break else: self._attributes.append(attribute) if isinstance(spec, str): tier_segs = self.features_to_segments(spec) else: tier_segs = spec for word in self: v = sum([1 for x in getattr(word, sequence_type) if x in tier_segs]) setattr(word, attribute.name, v) attribute.update_range(v) def add_tier(self, attribute, spec): """ Add a Tier Attribute based on the transcription of words as a new Attribute that includes all segments that match the specification. The specification should be either a list of segments or a string of the format '+feature1,-feature2' that specifies the set of segments. Parameters ---------- attribute : Attribute Attribute to add or replace spec : list or str Specification of what segments should be counted """ if isinstance(attribute,str): attribute = Attribute(attribute, 'tier') for i,a in enumerate(self._attributes): if attribute.name == a.name: self._attributes[i] = attribute break else: self._attributes.append(attribute) if isinstance(spec, str): tier_segs = self.features_to_segments(spec) else: tier_segs = spec attribute._range = tier_segs for word in self: word.add_tier(attribute.name,tier_segs) def remove_word(self, word_key): """ Remove a Word from the Corpus using its identifier in the Corpus. If the identifier is not found, nothing happens. Parameters ---------- word_key : string Identifier to use to remove the Word """ try: del self.wordlist[word_key] except KeyError: pass def remove_attribute(self, attribute): """ Remove an Attribute from the Corpus and from all its Word objects. Parameters ---------- attribute : Attribute Attribute to remove """ if isinstance(attribute,str): name = attribute else: name = attribute.name if name in self.basic_attributes: return for i in range(len(self._attributes)): if self._attributes[i].name == name: del self._attributes[i] break else: return for word in self: word.remove_attribute(name) def __getstate__(self): state = self.__dict__ return state def __setstate__(self, state): try: if 'inventory' not in state: state['inventory'] = state['_inventory'] #backcompat if not isinstance(state['inventory'], Inventory): state['inventory'] = Inventory() if 'has_spelling' not in state: state['has_spelling'] = state['has_spelling_value'] if 'has_transcription' in state: del state['has_transcription'] if 'has_wordtokens' not in state: state['has_wordtokens'] = False if '_freq_base' in state: del state['_freq_base'] if '_attributes' not in state: state['_attributes'] = [Attribute('spelling','spelling'), Attribute('transcription','tier'), Attribute('frequency','numeric')] try: tiers = state.pop('_tiers') for t in tiers: state['_attributes'].append(Attribute(t,'tier')) except KeyError: pass self.__dict__.update(state) #Backwards compatability for k,w in self.wordlist.items(): w._corpus = self # for a in self.attributes: # if a.att_type == 'tier': # if not isinstance(getattr(w,a.name), Transcription): # setattr(w,a.name,Transcription(getattr(w,a.name))) # else: # try: # a.update_range(getattr(w,a.name)) # except AttributeError as e: # print(k) # print(w.__dict__) # raise(e) except Exception as e: raise(e) raise(CorpusIntegrityError("An error occurred while loading the corpus: {}." "\nPlease redownload or recreate the corpus.".format(str(e)))) def _specify_features(self): self.inventory.specify(self.specifier) def check_coverage(self): """ Checks the coverage of the specifier (FeatureMatrix) of the Corpus over the inventory of the Corpus Returns ------- list List of segments in the inventory that are not in the specifier """ if not self.specifier is not None: return [] return [x for x in self.inventory.keys() if x not in self.specifier] def iter_words(self): """ Sorts the keys in the corpus dictionary, then yields the values in that order Returns ------- generator Sorted Words in the corpus """ sorted_list = sorted(self.wordlist.keys()) for word in sorted_list: yield self.wordlist[word] def iter_sort(self): """ Sorts the keys in the corpus dictionary, then yields the values in that order Returns ------- generator Sorted Words in the corpus """ sorted_list = sorted(self.wordlist.keys()) for word in sorted_list: yield self.wordlist[word] def set_feature_matrix(self,matrix): """ Set the feature system to be used by the corpus and make sure every word is using it too. Parameters ---------- matrix : FeatureMatrix New feature system to use in the corpus """ self.specifier = matrix def get_random_subset(self, size, new_corpus_name='randomly_generated'): """Get a new corpus consisting a random selection from the current corpus Parameters ---------- size : int Size of new corpus new_corpus_name : str Returns ------- new_corpus : Corpus New corpus object with len(new_corpus) == size """ new_corpus = Corpus(new_corpus_name) while len(new_corpus) < size: word = self.random_word() new_corpus.add_word(word, allow_duplicates=False) new_corpus.specifier = self.specifier return new_corpus def add_word(self, word, allow_duplicates=False): """Add a word to the Corpus. If allow_duplicates is True, then words with identical spelling can be added. They are kept sepearate by adding a "silent" number to them which is never displayed to the user. If allow_duplicates is False, then duplicates are simply ignored. Parameters ---------- word : Word Word object to be added allow_duplicates : bool If False, duplicate Words with the same spelling as an existing word in the corpus will not be added """ word._corpus = self tokens = word.wordtokens[:] # only becomes relevant when same spelling but different transcription? #If the word doesn't exist, add it try: check = self.find(word.spelling) if allow_duplicates: #Some words have more than one entry in a corpus, e.g. "live" and "live" #so they need to be assigned unique keys n = 0 while True: n += 1 key = '{} ({})'.format(word.spelling, n) try: check = self.find(key) except KeyError: if word.frequency == 0: word.frequency += 1 self.wordlist[key] = word break else: check.frequency += word.frequency if word.frequency != 0 else 1 check.wordtokens.extend(tokens) return except KeyError: if word.frequency == 0: word.frequency += 1 self.wordlist[word.spelling] = word #copy.copy(word) if word.spelling is not None: if not self.has_spelling: self.has_spelling = True added_default = False if word.transcription is not None: added_default = self.update_inventory(word.transcription) #added_default == True if the word contains symbols not found in the feature file #in this case, the symbol has been given a default value of 'n' for every feature word.transcription._list = [self.inventory[x].symbol for x in word.transcription._list] for d in word.descriptors: if d not in self._attributes: if isinstance(getattr(word, d), str): self._attributes.append(Attribute(d,'spelling'))#'factor')) elif isinstance(getattr(word, d), Transcription): self._attributes.append(Attribute(d,'tier')) elif isinstance(getattr(word, d), (int, float)): if word._freq_name not in self._attributes: self._attributes.append(Attribute(d, 'frequency')) for a in self._attributes: if not hasattr(word, a.name): word.add_attribute(a.name, a.default_value) a.update_range(getattr(word, a.name)) return added_default def update_features(self): for seg in self.inventory: if seg.symbol == '#': continue self.inventory[seg.symbol].features = self.specifier.specify(seg) def update_inventory(self, transcription): """ Update the inventory of the Corpus to ensure it contains all the segments in the given transcription Parameters ---------- transcription : list Segment symbols to add to the inventory if needed """ added_default = False for s in transcription: if isinstance(s, Segment): s = s.symbol if s not in self.inventory: self.inventory.segs[s] = Segment(s) if self.specifier is not None: if s not in self.specifier: self.specifier[s] = {feature.lower(): 'n' for feature in self.specifier.features} added_default = True self.inventory.segs[s].features = self.specifier[s] if transcription.stress_pattern: for k, v in transcription.stress_pattern.items(): self.inventory.stresses[v].add(transcription[k]) return added_default def get_or_create_word(self, **kwargs): """ Get a Word object that has the spelling and transcription specified or create that Word, add it to the Corpus and return it. Parameters ---------- spelling : string Spelling to search for transcription : list Transcription to search for Returns ------- Word Existing or newly created Word with the spelling and transcription specified """ try: spelling = kwargs['spelling'] except KeyError: for key,value in kwargs.items(): att_type = value[0].att_type default = value[0].is_default if att_type == 'spelling' and default: spelling = value[1] break else: return None words = self.find_all(spelling) for w in words: for k,v in kwargs.items(): if isinstance(v,tuple): v = v[1] if isinstance(v,list): v = Transcription(v) if getattr(w,k) != v: break else: return w else: word = Word(**kwargs) self.add_word(word) return word def random_word(self): """Return a randomly selected Word Returns ------- Word Random Word """ word = random.choice(list(self.wordlist.keys())) return self.wordlist[word] def get_features(self): """ Get a list of the _features used to describe Segments Returns ---------- list of str """ return self.specifier.features def find(self, word, ignore_case = False): """Search for a Word in the corpus Parameters ---------- word : str String representing the spelling of the word (not transcription) Returns ------- Word Word that matches the spelling specified Raises ------ KeyError If word is not found """ patterns = [word] if ignore_case: patterns.append(word.lower()) patterns.append(word.title()) for w in patterns: key = w try: result = self.wordlist[w] return result except KeyError: try: key = '{} (1)'.format(w) result = self.wordlist[key] return result except KeyError: pass raise KeyError('The word \"{}\" is not in the corpus'.format(word)) def find_all(self, spelling): """ Find all Word objects with the specified spelling Parameters ---------- spelling : string Spelling to look up Returns ------- list of Words Words that have the specified spelling """ words = list() try: words.append(self.wordlist[spelling]) count = 0 while True: count += 1 try: words.append(self.wordlist['{} ({})'.format(spelling,count)]) except KeyError: break except KeyError: pass return words def __contains__(self,item): return self.wordlist.__contains__(item) def __len__(self): return len(self.wordlist) def __setitem__(self,item,value): self.wordlist[item] = value def __getitem__(self,item): return self.wordlist[item] def __iter__(self): return iter(self.wordlist.values())