数据标注与数据集工程

import numpy as np

def cohen_kappa(a, b):
    # a, b 为两位标注者的标签序列
    a, b = np.asarray(a), np.asarray(b)
    po = (a == b).mean()
    classes = np.unique(np.concatenate([a, b]))
    pe = sum((a == c).mean() * (b == c).mean() for c in classes)
    return (po - pe) / (1 - pe + 1e-12)

labels_a = [0, 1, 1, 0, 1, 1, 0]
labels_b = [0, 1, 0, 0, 1, 1, 1]
print("kappa:", round(cohen_kappa(labels_a, labels_b), 4))

import numpy as np

def select_uncertain(confidences, k=10):
    # 置信度越接近 0.5 越不确定，优先标注
    uncertainty = 1 - np.abs(np.asarray(confidences) - 0.5) * 2
    order = np.argsort(-uncertainty)
    return order[:k].tolist()

conf = [0.9, 0.51, 0.99, 0.48, 0.75, 0.5]
print("to label:", select_uncertain(conf, k=3))

import json
from collections import Counter
import numpy as np
from pathlib import Path

def analyze_coco(coco_json):
    data = json.loads(Path(coco_json).read_text())
    cats = {c["id"]: c["name"] for c in data["categories"]}
    counts = Counter(a["category_id"] for a in data["annotations"])
    areas = np.array([a["area"] for a in data["annotations"]])

    print("images:", len(data["images"]), "annotations:", len(data["annotations"]))
    for cid, n in counts.most_common():
        print(f"{cats[cid]:<20s} {n}")
    print("area p50/p90/max:",
          round(float(np.percentile(areas, 50)), 1),
          round(float(np.percentile(areas, 90)), 1),
          round(float(areas.max()), 1))
    small = (areas < 32 ** 2).mean()
    print("small object ratio:", round(float(small), 4))

analyze_coco("annotations/instances_train2017.json")

import json
import xml.etree.ElementTree as ET
from pathlib import Path

def voc_to_coco(xml_dir, out_json, class_names):
    images, annotations, categories = [], [], []
    cat_id = {name: i + 1 for i, name in enumerate(class_names)}
    for i, name in enumerate(class_names):
        categories.append({"id": i + 1, "name": name})

    ann_id = 1
    for img_id, xml_path in enumerate(sorted(Path(xml_dir).glob("*.xml")), start=1):
        root = ET.parse(xml_path).getroot()
        w = int(root.find("size/width").text)
        h = int(root.find("size/height").text)
        images.append({"id": img_id, "file_name": root.find("filename").text,
                       "width": w, "height": h})
        for obj in root.findall("object"):
            name = obj.find("name").text
            box = obj.find("bndbox")
            x1 = float(box.find("xmin").text)
            y1 = float(box.find("ymin").text)
            x2 = float(box.find("xmax").text)
            y2 = float(box.find("ymax").text)
            annotations.append({
                "id": ann_id, "image_id": img_id, "category_id": cat_id[name],
                "bbox": [x1, y1, x2 - x1, y2 - y1],
                "area": (x2 - x1) * (y2 - y1), "iscrowd": 0,
            })
            ann_id += 1

    coco = {"images": images, "annotations": annotations, "categories": categories}
    Path(out_json).write_text(json.dumps(coco))
    print("images:", len(images), "annotations:", len(annotations))

voc_to_coco("VOCdevkit/Annotations", "coco.json", ["person", "car", "dog"])

import hashlib
from pathlib import Path

def file_hash(path, chunk=1 << 20):
    h = hashlib.md5()
    with open(path, "rb") as f:
        while chunk_data := f.read(chunk):
            h.update(chunk_data)
    return h.hexdigest()

def find_duplicates(image_dir):
    seen, dups = {}, []
    for p in Path(image_dir).rglob("*"):
        if p.suffix.lower() in {".jpg", ".png", ".jpeg"}:
            h = file_hash(p)
            if h in seen:
                dups.append((str(seen[h]), str(p)))
            else:
                seen[h] = p
    return dups

print("duplicates:", find_duplicates("images/train")[:5])