Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add per-label thresholds #2366

Merged
merged 5 commits into from
Aug 17, 2021
Merged

Add per-label thresholds #2366

Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ed26797
First commit for PR that adds per-label thresholds
yosipk Aug 10, 2021
1e0f783
Refactored so it's robust to setting the threshold from outside of th…
yosipk Aug 10, 2021
dd5f001
Added property and property getter for multi_label_threshold
yosipk Aug 10, 2021
45e5d8a
Fixed typo
yosipk Aug 10, 2021
e93de1e
Dummy commit to trigger push to PR
yosipk Aug 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 24 additions & 13 deletions flair/nn/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,7 @@ def _calculate_loss(self, scores, labels):

return self.loss_function(scores, labels), len(labels)


def predict(
self,
sentences: Union[List[Sentence], Sentence],
Expand Down Expand Up @@ -476,17 +477,17 @@ def predict(
if len(label_candidates) > 0:

if self.multi_label or multi_class_prob:
sigmoided = torch.sigmoid(scores)
s_idx = 0
for sentence, label in zip(sentences, label_candidates):
for idx in range(sigmoided.size(1)):
if sigmoided[s_idx, idx] > self.multi_label_threshold or multi_class_prob:
label_value = self.label_dictionary.get_item_for_index(idx)
if label_value == 'O': continue
label.set_value(value=label_value, score=sigmoided[s_idx, idx].item())
sigmoided = torch.sigmoid(scores) # size: (n_sentences, n_classes)
n_labels = sigmoided.size(1)
for s_idx, (sentence, label) in enumerate(zip(sentences, label_candidates)):
for l_idx in range(n_labels):
label_value = self.label_dictionary.get_item_for_index(l_idx)
if label_value == 'O': continue
label_threshold = self._get_label_threshold(label_value)
label_score = sigmoided[s_idx, l_idx].item()
if label_score > label_threshold or multi_class_prob:
label.set_value(value=label_value, score=label_score)
sentence.add_complex_label(label_name, copy.deepcopy(label))
s_idx += 1

else:
softmax = torch.nn.functional.softmax(scores, dim=-1)
conf, idx = torch.max(softmax, dim=-1)
Expand All @@ -503,6 +504,14 @@ def predict(
if return_loss:
return overall_loss, label_count

def _get_label_threshold(self, label_value):
if type(self.multi_label_theshold) is not map:
label_threshold = self.multi_label_threshold
else:
label_threshold = self.multi_label_threshold['default'] if label_value not in self.multi_label_threshold else self.multi_label_threshold[label_value]
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the multi_label_threshold does not have key default this fails. This should be checked anywhere where self.multi_label_threshold is set (in constructor, when saving or loading the model?)

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Possibility is to have multi_label_threshold as a property, and check for this stuff in setter. Example code:

class Model:

    def __init__(self, threshold):
        self.threshold = threshold

    @property
    def threshold(self):
        return self._threshold

    @threshold.setter
    def threshold(self, x):
        if type(x) is not dict:
            raise Exception('Not a dict')
        elif 'default' not in x:
            raise Exception('default key not present')
        else:
            self._threshold = x


return label_threshold

def _obtain_labels(
self, scores: List[List[float]], predict_prob: bool = False
) -> List[List[Label]]:
Expand All @@ -526,9 +535,11 @@ def _get_multi_label(self, label_scores) -> List[Label]:

results = list(map(lambda x: sigmoid(x), label_scores))
for idx, conf in enumerate(results):
if conf > self.multi_label_threshold:
label = self.label_dictionary.get_item_for_index(idx)
labels.append(Label(label, conf.item()))
label_value = self.label_dictionary.get_item_for_index(idx)
label_threshold = self._get_label_threshold(label_value)
label_score = conf.item()
if label_score > label_threshold:
labels.append(Label(label_value, label_score))

return labels

Expand Down