{ "generated_from": "data/", "stats": { "papers": 384, "verified": 384, "full_text": 108, "families": 15, "settings": 10, "problems": 10, "newest_paper": "2026-08-04", "by_year": { "2009": 1, "2015": 1, "2017": 1, "2018": 2, "2019": 3, "2020": 7, "2021": 10, "2022": 38, "2023": 73, "2024": 97, "2025": 88, "2026": 63 }, "built_on": "2026-09-14" }, "paradigms": [ { "id": "selection", "code": "P0", "name": "Selection", "question": "Which real samples should be kept?", "one_line": "The distilled set is a subset of the real data, chosen by a scoring or covering rule.", "coupling": "none — no synthesis" }, { "id": "bilevel", "code": "P1", "name": "Bi-level performance matching", "question": "Does a model trained on the synthetic set perform like one trained on the real set?", "one_line": "Optimize the synthetic set through the training of a model on it, by unrolling or in closed form.", "coupling": "the full inner training loop is differentiated (or solved analytically)" }, { "id": "surrogate-matching", "code": "P2", "name": "Surrogate matching", "question": "Does the synthetic set produce the same gradients, trajectories, or features?", "one_line": "Replace the bi-level objective with a single-level proxy computed through networks in the loop.", "coupling": "one network (or a short segment of its training) per update" }, { "id": "decoupled", "code": "P3", "name": "Decoupled synthesis", "question": "Can a frozen teacher alone tell us what the synthetic set should contain?", "one_line": "Train a teacher once, synthesize or select from it without any bi-level coupling, then train students on its soft labels.", "coupling": "a frozen pretrained teacher; no student in the synthesis loop" }, { "id": "generative", "code": "P4", "name": "Generative priors", "question": "Can a pretrained generator supply the space, the artifact, or the sampler?", "one_line": "Use a pretrained generative model to parameterize, store, or directly sample the synthetic set.", "coupling": "a frozen (or lightly fine-tuned) generator; often no learner in the loop at all" }, { "id": "orthogonal", "code": "+", "name": "Orthogonal design choices", "question": "What is stored, what is the label, and how is it optimized?", "one_line": "Choices that attach to a method in any paradigm — parameterization, label handling, optimization recipes.", "coupling": "independent of paradigm" }, { "id": "lens", "code": "?", "name": "Understanding and trust", "question": "What does condensation actually learn, and can we trust how it is measured?", "one_line": "Theory, evaluation, benchmarks, privacy, robustness and fairness of condensed data.", "coupling": "not a synthesis method" } ], "settings": [ { "id": "image-classification", "type": "setting", "name": "Image classification", "short": "Images", "group": "unimodal", "order": 1, "one_line": "The canonical setting. Condense a labeled image dataset so that a freshly initialized classifier trained only on the synthetic set approaches the accuracy of one trained on the full data.", "sample": "An image, or a stored parameterization that decodes to images, with a hard, learned or soft class label.", "evaluation": "Train a network from scratch on the synthetic set (a 3–4 layer ConvNet on small benchmarks, ResNet-18/50 on ImageNet-1K) and report top-1 accuracy on the real test set, averaged over several training runs. Cross-architecture transfer is reported separately.", "budget_unit": "images per class (IPC)", "benchmarks": [ "MNIST / Fashion-MNIST", "CIFAR-10 / CIFAR-100", "Tiny-ImageNet", "ImageNet subsets (ImageNette", "ImageWoof", "…)", "ImageNet-1K", "ImageNet-21K" ], "what_is_different": [ "The budget is counted per class, so class-conditional structure comes for free: almost every method matches statistics, gradients or trajectories class by class.", "There are two scale regimes with different winners. On CIFAR-scale benchmarks with hard labels, surrogate-matching methods (P2) set the pace for years. At ImageNet-1K scale, decoupled (P3) and generative (P4) methods with teacher soft labels are the ones that run at all.", "Reported accuracy depends on the evaluation recipe (epochs, augmentation, the soft-label teacher) nearly as much as on the synthetic images. The evaluation page treats this as a first-class issue." ], "pitfalls": [ "Comparing a result evaluated with soft labels from a pretrained teacher against one evaluated with hard labels.", "Comparing numbers obtained with different evaluation networks, epochs or augmentation pipelines.", "Reporting accuracy only on the architecture used during distillation." ], "key_papers": [ "dd-2018", "dc-2021", "kip-2021", "mtt-2022", "dm-2023", "idc-2022", "sre2l-2023", "rded-2024", "glad-2023", "d4m-2024", "minimax-2024", "label-worth-2024", "dc-bench-2022" ], "explanation": null }, { "id": "vision-language", "type": "setting", "name": "Image–text pairs (vision–language)", "short": "Vision–language", "group": "multimodal", "order": 2, "one_line": "Condense a paired image–caption corpus so that a dual encoder trained on the synthetic pairs retrieves nearly as well as one trained on the full corpus.", "sample": "An image paired with one or more captions (often stored as text embeddings), sometimes accompanied by a learned image–text similarity structure instead of strict one-to-one pairing.", "evaluation": "Train or fine-tune an image encoder and a text encoder or projection contrastively on the synthetic pairs, then report image→text and text→image Recall@K on the Flickr30K or MS-COCO test split.", "budget_unit": "number of image–text pairs (typically 100–1000)", "benchmarks": [ "Flickr30K", "MS-COCO" ], "what_is_different": [ "There are no classes. The target is the joint image–text distribution, so class-wise matching has no direct analogue, and the correspondence between the two modalities is itself something the distilled set has to preserve.", "The downstream learner is contrastive, so the value of one pair depends on the pairs it is contrasted against. Distilling pairwise similarity structure rather than assuming one caption per image is one response to this.", "Methods split into optimization-based ones (trajectory, distribution, covariance or analytic parameter matching through the encoders) and generative ones (a pretrained diffusion prior synthesizes the images, conditioned on real, prototypical or generated text).", "Text is usually distilled in embedding space, so the released artifact is not always a readable dataset." ], "pitfalls": [ "Evaluation protocols are not shared across papers: pretrained versus frozen image backbones, frozen BERT versus CLIP text encoders, trainable versus fixed text towers. Numbers from different protocols are not comparable, and the difference for a single method can be large.", "Extra stored structure (similarity matrices, soft labels) is rarely counted against the pair budget.", "Recall at 100–1000 pairs remains far below full-data recall; gains over weak coreset baselines can overstate practical usefulness." ], "key_papers": [ "mtt-vl-2024", "lors-2024", "repblend-2025", "covmatch-2025", "edge-2025", "pds-2026", "mdm-2026", "raha-2026" ], "explanation": null }, { "id": "audio-visual-omni", "type": "setting", "name": "Audio–visual and omnimodal data", "short": "Audio–visual & omni", "group": "multimodal", "order": 3, "one_line": "Condense datasets whose samples span audio, vision and sometimes further modalities, while preserving the correspondence between them.", "sample": "A labeled (visual frame, audio) pair, or a tuple of modality embeddings in a shared bound space.", "evaluation": "Train an audio–visual classifier or retrieval model on the synthetic set and evaluate on the real test split.", "budget_unit": "samples per class", "benchmarks": [ "VGGSound subsets", "AVE", "Music-21", "AudioSet subsets" ], "what_is_different": [ "Matching each modality's distribution separately can destroy the alignment between them, so methods add explicit cross-modal terms.", "Omnimodal work condenses in a shared embedding space that binds more than two modalities, which turns the problem into preserving higher-order relations, not only pairwise ones." ], "pitfalls": [ "The literature is small; baselines are often unimodal methods applied per modality, which sets a low bar for a joint method." ], "key_papers": [ "avdd-2024", "davdd-2025", "imagebinddc-2026" ], "explanation": null }, { "id": "video", "type": "setting", "name": "Video", "short": "Video", "group": "other-modalities", "order": 4, "one_line": "Condense video datasets for action recognition or temporal understanding, where most of the cost and most of the redundancy lie in the time axis.", "sample": "A short synthetic clip, or a static image combined with a separately stored dynamic component.", "evaluation": "Train an action-recognition model on the synthetic clips and report accuracy on UCF101, HMDB51, Kinetics or Something-Something subsets.", "budget_unit": "clips per class", "benchmarks": [ "UCF101", "HMDB51", "Kinetics-400 subsets", "Something-Something V2" ], "what_is_different": [ "Appearance and motion carry different amounts of information per frame, which motivates storing them separately rather than distilling every frame.", "Memory for backpropagating through clips limits synthetic clip length and resolution." ], "pitfalls": [ "Very short synthetic clips and small evaluation backbones make it hard to tell whether motion, not appearance, was preserved." ], "key_papers": [ "video-sdd-2024", "video-dc-study-2024" ], "explanation": null }, { "id": "graph", "type": "setting", "name": "Graphs", "short": "Graphs", "group": "other-modalities", "order": 5, "one_line": "Graph condensation: synthesize a small graph (or a few small graphs) on which a graph neural network trains to the accuracy it reaches on the original large graph.", "sample": "Synthetic node features and labels with a synthetic or omitted adjacency structure.", "evaluation": "Train a GNN (GCN, SGC, APPNP, …) on the condensed graph and evaluate node classification on the original graph (Cora, Citeseer, ogbn-arxiv, Flickr, Reddit) or graph classification on TU datasets.", "budget_unit": "condensation ratio (fraction of nodes kept)", "benchmarks": [ "Cora", "Citeseer", "ogbn-arxiv", "Flickr", "Reddit", "TU graph-classification datasets" ], "what_is_different": [ "Structure must either be synthesized jointly with features or dropped entirely; structure-free condensation shows the latter can work.", "Matching passes through message passing, so the condensed graph is tied to the propagation scheme.", "The subfield has its own surveys and benchmarks and is often organized alongside graph sparsification and coarsening." ], "pitfalls": [ "Transductive and inductive settings, and differing definitions of the condensation ratio, make results hard to compare." ], "key_papers": [ "gcond-2022", "doscond-2022", "sfgc-2023", "geom-2024", "gc-bench-2024", "survey-graph-condensation-2025" ], "explanation": null }, { "id": "text", "type": "setting", "name": "Text and language models", "short": "Text", "group": "other-modalities", "order": 6, "one_line": "Condense text datasets for classifiers or language-model fine-tuning, where the discreteness of tokens breaks the gradient-based synthesis that works for pixels.", "sample": "A token sequence, a sequence of embeddings, or soft token distributions.", "evaluation": "Fine-tune a text classifier or language model on the synthetic set and evaluate on the real test set.", "budget_unit": "examples per class", "benchmarks": [ "SST-2", "AG News", "other GLUE-style classification tasks", "instruction / fine-tuning corpora" ], "what_is_different": [ "Tokens are discrete, so methods either distill in embedding space (efficient but not human readable), attach distilled attention labels, or distill into a generator language model that emits readable text.", "In the LLM era, condensation meets synthetic-data generation for fine-tuning." ], "pitfalls": [ "Embedding-space synthetic sets only work with the embedding table they were distilled for." ], "key_papers": [ "sldd-2021", "dilm-2024", "grad-mm-2025" ], "explanation": null }, { "id": "time-series", "type": "setting", "name": "Time series and spatio-temporal data", "short": "Time series", "group": "other-modalities", "order": 7, "one_line": "Condense sequential datasets for forecasting or classification, where the target is often a regression output rather than a class label.", "sample": "A synthetic multivariate sequence or window.", "evaluation": "Train a forecasting model on the synthetic set and report MSE/MAE on real test windows, or accuracy for classification.", "budget_unit": "number of synthetic sequences or windows", "benchmarks": [ "ETT", "Weather", "Electricity", "Traffic", "UCR/UEA classification archives" ], "what_is_different": [ "Forecasting is regression, so class-conditional matching does not apply directly.", "Frequency-domain and shape-level structure (periodicity, shapelets) are natural things to match." ], "pitfalls": [ "Forecasting benchmarks are small and saturated; gains over a random subset can be within run-to-run variance." ], "key_papers": [ "timedc-2024", "condtsf-2024" ], "explanation": null }, { "id": "self-supervised", "type": "setting", "name": "Pre-training, transfer and foundation models", "short": "Pre-training & transfer", "group": "beyond-classification", "order": 8, "one_line": "Condense unlabeled data for self-supervised pre-training, or condense data for training on top of frozen pretrained models, and judge the result by transfer rather than by in-distribution accuracy.", "sample": "An unlabeled image, often paired with a target representation from a pretrained model.", "evaluation": "Pre-train on the synthetic set, then linear-probe or fine-tune on downstream tasks; or train a linear head on frozen foundation-model features.", "budget_unit": "number of images", "benchmarks": [ "CIFAR / Tiny-ImageNet / ImageNet as source", "downstream classification transfer suites" ], "what_is_different": [ "There are no labels to condition on, and naive bi-level optimization through a self-supervised objective is unstable.", "Representation targets from a pretrained model turn the problem into regression on features." ], "pitfalls": [ "Transfer results depend strongly on the downstream protocol and on which pretrained model supplied the targets." ], "key_papers": [ "krr-st-2024", "rela-2024", "mkdt-2025", "linear-gradient-matching-2025" ], "explanation": null }, { "id": "dense-prediction", "type": "setting", "name": "Detection, segmentation and low-level vision", "short": "Dense prediction", "group": "beyond-classification", "order": 9, "one_line": "Condense datasets whose labels are spatial (boxes, masks, target images), where one image holds many objects and synthesis must preserve layout.", "sample": "An image with boxes, masks, or a paired target image.", "evaluation": "Train a detector, segmenter or restoration network on the synthetic set; report mAP, mIoU or PSNR/SSIM.", "budget_unit": "number of images, or a fraction of the training set", "benchmarks": [ "PASCAL VOC", "MS-COCO detection", "Cityscapes / ADE20K", "super-resolution and deraining benchmarks" ], "what_is_different": [ "A class-wise budget is ill-defined when an image contains several classes.", "Synthesis must keep object layout and scale statistics, not just class appearance." ], "pitfalls": [ "Very few methods and no shared protocol; results are mostly compared against random subsets." ], "key_papers": [ "dcod-2024", "od3-2026" ], "explanation": null }, { "id": "other-data", "type": "setting", "name": "Other data types", "short": "Other data", "group": "other-modalities", "order": 10, "one_line": "Point clouds, tabular data, recommender interactions, speech, EEG, event streams, reinforcement learning and more, each with a handful of papers adapting condensation to its structure.", "sample": "Domain-specific (a point set, a table row, a user–item matrix, a spectrogram, a state–action set, …).", "evaluation": "Domain-specific; always training a model on the synthetic data and testing on real data.", "budget_unit": "domain-specific", "benchmarks": [ "ModelNet / ScanObjectNN", "recommendation datasets", "speech emotion corpora", "EEG corpora", "RL environments" ], "what_is_different": [ "Each domain has an invariance that images lack (permutation for point sets, sparsity for interaction matrices, temporal dynamics for RL) that the synthesis must respect." ], "pitfalls": [ "Most domains have one to three papers; treat claims as early evidence." ], "key_papers": [ "distill-cf-2022", "point-cloud-dd-2025" ], "explanation": null } ], "families": [ { "id": "coreset-selection", "type": "family", "name": "Coreset selection", "paradigm": "selection", "order": 1, "one_line": "Keep a subset of real samples chosen by a scoring or covering rule; no pixels are synthesized.", "core_bet": "A well-chosen subset of real data captures enough of what training needs, and staying on the data manifold is worth more than the flexibility of synthesis.", "formulation": "\\mathcal{S}^{\\star}=\\arg\\min_{\\mathcal{S}\\subseteq\\mathcal{T},\\;|\\mathcal{S}|=m}\\;\\max_{x\\in\\mathcal{T}}\\;\\min_{s\\in\\mathcal{S}}\\;\\bigl\\|\\phi(x)-\\phi(s)\\bigr\\|_2", "formulation_note": "The k-center (covering) criterion, one of several. Herding instead greedily matches the class mean of features; forgetting and error/gradient-norm scores rank samples by training dynamics.", "since": 2009, "status": "mature", "branches": [ { "id": "geometric", "name": "Geometric criteria", "description": "Herding (mean matching) and k-center (covering radius) in a feature space." }, { "id": "training-dynamics", "name": "Training-dynamics criteria", "description": "Forgetting events, early-training error or gradient norms, and similar difficulty scores." } ], "strengths": [ "Produces real, interpretable samples that any architecture can train on.", "Cheap, and needs no optimization over pixels.", "The honest baseline for every condensation paper; random selection is the floor any method must clear." ], "weaknesses": [ "A subset cannot interpolate between examples or pack more information into one sample than one real image carries.", "At very small budgets the gap to synthesized sets is large on standard benchmarks." ], "competes_with": [ "distribution-matching", "decoupled-synthesis", "dataset-quantization" ], "explanation": null, "arc": [ { "paper": "herding-2009", "role": "origin", "note": "Herding greedily picks samples so that the running mean of selected features tracks the class mean, a first-moment criterion. It later became the standard selection baseline in condensation papers because it optimizes the same statistic distribution matching does, restricted to real images." }, { "paper": "kcenter-coreset-2018", "role": "origin", "note": "Casts selection as covering the data in feature space and chooses points that minimize the largest distance from any sample to its nearest selected one, derived from a generalization bound. The geometric baseline that recurs throughout condensation benchmarks." }, { "paper": "forgetting-2019", "role": "origin", "note": "Ranks samples by training dynamics: examples that are learned and never forgotten are redundant. On CIFAR-10, 30% of the training set can be removed this way with accuracy comparable to full-data training, far better than removing a random 30%." }, { "paper": "data-diet-2021", "role": "improvement", "note": "Scores sample importance early in training (gradient and error norms), making training-dynamics selection much cheaper than tracking forgetting over a full run." }, { "paper": "selmatch-2024", "role": "extension", "note": "Brings selection into synthesis: initializes a trajectory-matched set from selected real images and updates only part of it, because pure synthesis falls below random selection as images per class grow." } ] }, { "id": "dataset-quantization", "type": "family", "name": "Dataset quantization", "paradigm": "selection", "order": 2, "one_line": "Partition the dataset into non-overlapping bins by a diversity criterion and sample from every bin, optionally dropping uninformative patches.", "core_bet": "Coverage of the whole distribution matters more than per-sample informativeness, and a selection that is not tied to one architecture transfers across training recipes.", "since": 2023, "status": "active", "strengths": [ "Keeps real samples and is architecture-agnostic, including for pre-training-style recipes.", "Scales to large datasets because no bi-level optimization is involved." ], "weaknesses": [ "Compression is bounded by what real samples can express; very small budgets remain hard." ], "competes_with": [ "coreset-selection" ], "explanation": null, "arc": [ { "paper": "dq-2023", "role": "origin", "note": "Compresses large datasets into subsets usable by any architecture, reporting training with little or no loss from 60% of ImageNet and 20% of Alpaca instruction data across vision and language tasks." } ] }, { "id": "meta-learning-bptt", "type": "family", "name": "Meta-learning through unrolled training", "paradigm": "bilevel", "order": 3, "one_line": "Treat the synthetic data as hyperparameters of training and optimize them by backpropagating through the unrolled inner optimization.", "core_bet": "The faithful objective — the real-data loss of a model trained on the synthetic set — is worth optimizing directly, if the cost of differentiating through training can be managed.", "formulation": "\\min_{\\mathcal{S}}\\;\\mathcal{L}\\bigl(\\theta_{T}(\\mathcal{S});\\,\\mathcal{T}\\bigr),\\qquad \\theta_{t+1}=\\theta_{t}-\\eta\\,\\nabla_{\\theta}\\,\\ell\\bigl(\\theta_{t};\\,\\mathcal{S}\\bigr)", "formulation_note": "The outer gradient flows through all T inner steps (backpropagation through time), or a truncated window of them.", "since": 2018, "status": "foundational", "strengths": [ "Optimizes the quantity that is actually evaluated, with no surrogate mismatch.", "Conceptually simple, and it defined the task." ], "weaknesses": [ "Memory and compute grow with the number of unrolled steps; long unrolls are also numerically unstable.", "Truncating the unroll biases the solution toward early-training behavior." ], "competes_with": [ "kernel-closed-form", "gradient-matching", "trajectory-matching" ], "explanation": null, "arc": [ { "paper": "dd-2018", "role": "origin", "note": "Poses the task. Synthetic points need not come from the data distribution; they only have to reproduce, through a few gradient steps from a fixed initialization, the model trained on the full data. MNIST's 60,000 training images are compressed into 10 synthetic images." }, { "paper": "rat-bptt-2024", "role": "improvement", "note": "Returns to backpropagation through time, traces its difficulty to gradient variance, compute and long-range dependencies, and stabilizes it with randomly placed truncation windows. Also finds that subsets of a distilled set perform much worse than a set distilled directly at the smaller size." } ] }, { "id": "kernel-closed-form", "type": "family", "name": "Kernel and closed-form inner solvers", "paradigm": "bilevel", "order": 4, "one_line": "Replace the inner training loop with kernel ridge regression (NTK, random features, or a trained feature extractor), so the inner problem has a closed-form solution.", "core_bet": "If the learner can be approximated by a kernel machine, the bi-level problem becomes a single differentiable expression and no unrolling is needed.", "formulation": "\\min_{X_s,\\,Y_s}\\;\\tfrac{1}{2}\\,\\Bigl\\|\\,Y_t-K_{X_tX_s}\\bigl(K_{X_sX_s}+\\lambda I\\bigr)^{-1}Y_s\\Bigr\\|_2^2", "formulation_note": "K is the kernel between target and support (synthetic) points. KIP uses the NTK; RFAD approximates it with random features; FRePo uses the features of a network trained alongside.", "since": 2021, "status": "mature", "strengths": [ "Exact inner solution with no truncation bias; strong results at very small budgets.", "Learned labels come naturally, since the support labels enter the closed form directly." ], "weaknesses": [ "Kernel matrices scale quadratically with the support set, and NTK computation is expensive for deep convolutional architectures.", "A gap remains between infinite-width kernels and the finite networks the data is finally used with." ], "competes_with": [ "meta-learning-bptt", "trajectory-matching" ], "explanation": null, "arc": [ { "paper": "kip-2021", "role": "origin", "note": "Kernel Inducing Points uses the correspondence between infinitely wide networks and kernel ridge regression to learn datasets one to two orders of magnitude smaller for KRR, and reports that the learned data transfers to finite-width networks." }, { "paper": "frepo-2022", "role": "improvement", "note": "Neural feature regression with a pool of models: a truncated alternative to differentiating through the whole inner loop, reported to need an order of magnitude less memory and to train two orders of magnitude faster, with results up to ImageNet-1K." }, { "paper": "rfad-2022", "role": "scaling", "note": "Replaces KIP's exact kernel matrix, quadratic in the synthetic set size, with a random-feature approximation of the NNGP kernel that is linear in it, for at least a 100-fold speedup on one GPU." }, { "paper": "rcig-2023", "role": "improvement", "note": "Computes meta-gradients by implicit differentiation on a convexified problem, learning on top of a frozen finite-width neural tangent kernel with analytically computed final-layer parameters." } ] }, { "id": "gradient-matching", "type": "family", "name": "Gradient matching", "paradigm": "surrogate-matching", "order": 5, "one_line": "Make the gradients a network computes on synthetic data match the gradients it computes on real data, step by step along a training run.", "core_bet": "If every training step on the synthetic set moves the parameters the way a step on real data would, training on the synthetic set follows the real-data trajectory — without unrolling it.", "formulation": "\\min_{\\mathcal{S}}\\;\\mathbb{E}_{\\theta_0\\sim P_{\\theta_0}}\\Bigl[\\sum_{t=0}^{T-1}\\sum_{c=1}^{C} D\\bigl(\\nabla_{\\theta}\\ell(\\theta_t;\\mathcal{S}_c),\\;\\nabla_{\\theta}\\ell(\\theta_t;\\mathcal{T}_c)\\bigr)\\Bigr]", "formulation_note": "D is a layer-wise distance (e.g. cosine-based); the network is updated on the synthetic set between matching steps. DSA applies the same differentiable augmentation to both batches.", "since": 2021, "status": "mature", "strengths": [ "Avoids unrolling; each update needs only one forward–backward pass on each batch.", "Its differentiable Siamese augmentation became part of the standard evaluation protocol." ], "weaknesses": [ "Single-step gradients are a short-horizon signal, and errors accumulate over the run.", "The nested loop over initializations and steps is still expensive, and results are tied to the matching architecture." ], "competes_with": [ "trajectory-matching", "distribution-matching", "kernel-closed-form" ], "explanation": null, "arc": [ { "paper": "dc-2021", "role": "origin", "note": "Formulates condensation as matching the gradients of network weights computed on synthetic and on real data, and demonstrates the condensed sets for continual learning and architecture search." }, { "paper": "dsa-2021", "role": "improvement", "note": "Applies the same differentiable augmentation to real and synthetic batches during matching, so that synthetic images stay informative under the augmentation used to train on them." }, { "paper": "dcc-2022", "role": "challenge", "note": "Proves that class-wise gradient matching can do worse than random selection when task-irrelevant information dominates the data, and restores contrastive signal between classes; the fix matters most for fine-grained classification." } ] }, { "id": "trajectory-matching", "type": "family", "name": "Trajectory matching", "paradigm": "surrogate-matching", "order": 6, "one_line": "Train a student for several steps on synthetic data from a point on a precomputed expert trajectory, and make it land where the expert landed after many more real-data steps.", "core_bet": "Matching long-range parameter movement is a better proxy for \"trains like the real data\" than matching single-step gradients, and expert trajectories can be computed once and reused.", "formulation": "\\min_{\\mathcal{S}}\\;\\frac{\\bigl\\|\\hat{\\theta}_{t+N}-\\theta^{*}_{t+M}\\bigr\\|_2^2}{\\bigl\\|\\theta^{*}_{t}-\\theta^{*}_{t+M}\\bigr\\|_2^2},\\qquad \\hat{\\theta}_{t+N}=\\text{$N$ SGD steps on }\\mathcal{S}\\text{ starting from }\\theta^{*}_{t}", "formulation_note": "θ* are checkpoints of experts trained on real data; N synthetic steps are matched against M expert steps, and the learning rate can itself be learned.", "since": 2022, "status": "mature", "branches": [ { "id": "accuracy", "name": "Accumulated error and difficulty alignment", "description": "Flatter or error-corrected trajectories, and matching early versus late expert segments according to the budget.", "papers": [ "ftd-2023", "datm-2024", "selmatch-2024", "pad-2024" ] }, { "id": "memory", "name": "Memory and scale", "description": "Constant-memory gradient computation and other approximations that make the unroll affordable at ImageNet scale.", "papers": [ "tesla-2023" ] } ], "strengths": [ "Among the strongest results on CIFAR- and Tiny-ImageNet-scale benchmarks, and the only family to reach full-data accuracy (with learned soft labels, at large budgets).", "Expert trajectories are computed once and shared across distillation runs." ], "weaknesses": [ "Training and storing many expert trajectories is a large up-front cost.", "Backpropagating through N student steps is memory-hungry, and the method is sensitive to N, M and the start-epoch range.", "Distilled sets are tied to the expert architecture more strongly than distribution-matched ones.", "Its most refined variants still lose to decoupled synthesis at low budgets on harder datasets, and its best numbers rely on learned soft labels, so gains are not purely from the images." ], "competes_with": [ "gradient-matching", "distribution-matching", "kernel-closed-form", "decoupled-synthesis" ], "explanation": null, "arc": [ { "paper": "mtt-2022", "role": "origin", "note": "Trains a network for several steps on the distilled data and minimizes its distance to the parameters that experts trained on real data reached, using expert trajectories precomputed once and stored. Also extends distillation to higher-resolution images." }, { "paper": "ftd-2023", "role": "improvement", "note": "Attributes part of the gap between distillation and evaluation to accumulated trajectory error, and regularizes the distillation toward flat trajectories." }, { "paper": "tesla-2023", "role": "scaling", "note": "Computes the unrolled gradient exactly with constant memory and adds a soft-label assignment for datasets with many classes, reaching 50 images per class on ImageNet-1K on a single GPU." }, { "paper": "datm-2024", "role": "improvement", "note": "Early expert trajectories suit small synthetic sets and late ones suit large sets; aligning trajectory difficulty with the set size keeps the method effective as the budget grows, and the paper reports lossless distillation for the first time." }, { "paper": "selmatch-2024", "role": "improvement", "note": "Shows that trajectory-matching methods can fall below random selection as images per class grow, because rare, hard features are not captured; initializes from selected real images and updates only part of the set." }, { "paper": "pad-2024", "role": "improvement", "note": "Filters misaligned information: prunes the target data according to the compression ratio and distills through deep layers only." } ] }, { "id": "distribution-matching", "type": "family", "name": "Distribution and feature matching", "paradigm": "surrogate-matching", "order": 7, "one_line": "Make the synthetic set's feature distribution match the real data's, in the embedding spaces of many (often randomly initialized) networks — with no bi-level loop at all.", "core_bet": "A synthetic set whose features are distributed like the real data's will train a model like the real data, and random networks already provide embeddings informative enough to compare the two.", "formulation": "\\min_{\\mathcal{S}}\\;\\mathbb{E}_{\\theta\\sim P_{\\theta}}\\sum_{c=1}^{C}\\Bigl\\|\\frac{1}{|\\mathcal{T}_c|}\\sum_{x\\in\\mathcal{T}_c}\\psi_{\\theta}(x)-\\frac{1}{|\\mathcal{S}_c|}\\sum_{s\\in\\mathcal{S}_c}\\psi_{\\theta}(s)\\Bigr\\|_2^2", "formulation_note": "The empirical maximum mean discrepancy with a linear kernel, i.e. class-wise mean embeddings. Later members change what is matched (attention maps, higher moments, style statistics, full characteristic functions, Wasserstein barycenters) or which networks provide the embedding.", "since": 2021, "status": "active", "branches": [ { "id": "richer-statistics", "name": "Richer statistics", "description": "Matching more than the first moment — kernel MMD, higher-order moments, style statistics, characteristic functions, optimal transport.", "papers": [ "m3d-2024", "wmdd-2025", "ncfm-2025" ] }, { "id": "better-embeddings", "name": "Better embedding networks", "description": "Partially trained or model-pool embeddings, layer-wise and attention features, and class-aware weighting.", "papers": [ "idm-2023", "datadam-2023", "dance-2024" ] } ], "strengths": [ "No inner training and no expert trajectories; orders of magnitude cheaper than bi-level methods.", "Scales comfortably to more classes and larger budgets, and is simple to extend to new modalities." ], "weaknesses": [ "At very small budgets, plain mean matching trails trajectory matching on standard benchmarks.", "Matching a class mean constrains one vector per class and leaves diversity, style and within-class structure free, which later members add back explicitly." ], "competes_with": [ "gradient-matching", "trajectory-matching", "decoupled-synthesis" ], "explanation": null, "arc": [ { "paper": "cafe-2022", "role": "precursor", "note": "Argues that gradient matching overfits to samples with dominant gradients and lacks global supervision of the data distribution, and aligns multi-scale features instead, still inside a bi-level scheme." }, { "paper": "dm-2023", "role": "origin", "note": "Removes bi-level optimization. Synthetic images match the feature distributions of real images in many sampled embedding spaces, which cuts synthesis cost enough to use larger datasets and architectures." }, { "paper": "idm-2023", "role": "improvement", "note": "Identifies imbalanced feature numbers and unvalidated embeddings as weaknesses of naive distribution matching, and adds partition-and-expand augmentation, richer model sampling and class-aware regularization." }, { "paper": "datadam-2023", "role": "improvement", "note": "Matches spatial attention maps from several layers of randomly initialized networks rather than pooled features only." }, { "paper": "m3d-2024", "role": "improvement", "note": "Argues that mean matching ignores higher-order alignment, and minimizes maximum mean discrepancy in a reproducing kernel Hilbert space so that all moments are aligned." }, { "paper": "dance-2024", "role": "improvement", "note": "Uses a few pretrained models to address distribution matching's limits within classes (a proxy for long-term training) and between classes (distribution shift out of the class region)." }, { "paper": "wmdd-2025", "role": "improvement", "note": "Matches the Wasserstein barycenter of features from a pretrained classifier and keeps per-class BatchNorm statistics to preserve intra-class variation." }, { "paper": "ncfm-2025", "role": "improvement", "note": "Recasts matching as a min-max problem over a neural characteristic-function discrepancy that captures the full distribution, and reports large memory and speed savings." } ] }, { "id": "decoupled-synthesis", "type": "family", "name": "Decoupled teacher-driven synthesis", "paradigm": "decoupled", "order": 8, "one_line": "Train one teacher on the real data, synthesize or select images from the teacher alone, then train students on the teacher's soft labels.", "core_bet": "The coupling between the synthetic data and a student being trained on it is what prevents scaling; a frozen teacher's batch-norm statistics, logits and features contain enough of the dataset to do without it.", "formulation": "\\min_{\\tilde{x}}\\;\\ell\\bigl(f_{\\theta_{\\mathcal{T}}}(\\tilde{x}),y\\bigr)+\\alpha\\sum_{l}\\Bigl(\\bigl\\|\\mu_l(\\tilde{x})-\\mathbb{E}[\\mu_l\\mid\\mathcal{T}]\\bigr\\|_2+\\bigl\\|\\sigma_l^2(\\tilde{x})-\\mathbb{E}[\\sigma_l^2\\mid\\mathcal{T}]\\bigr\\|_2\\Bigr)", "formulation_note": "The \"recover\" step of squeeze–recover–relabel: invert the teacher f while matching per-layer batch-norm running statistics. The \"relabel\" step then stores region-level soft labels from the teacher, which the student is trained against.", "since": 2023, "status": "ascendant", "branches": [ { "id": "inversion", "name": "Inversion from teacher statistics", "description": "Synthesize images by matching batch-norm statistics and class logits of one or several pretrained backbones, with curricula and diversity terms.", "papers": [ "sre2l-2023", "cda-2024", "g-vbsm-2024", "edc-2024", "dwa-2024", "delt-2025", "fadrm-2025" ] }, { "id": "realistic-selection", "name": "Realistic patch selection", "description": "Score and select the most representative real crops with the teacher and stitch them into synthetic images, trading optimization for realism and speed.", "papers": [ "rded-2024" ] } ], "strengths": [ "Made ImageNet-1K and ImageNet-21K condensation practical; synthesis cost does not depend on a student network.", "Distilled sets transfer across student architectures, helped by the teacher's soft labels." ], "weaknesses": [ "Much of the accuracy comes from the teacher's soft labels, which must be stored per crop and per epoch and can outweigh the images in storage.", "Inverted images can be unrealistic and low in diversity; realism-oriented variants give up some of the teacher's information.", "Hard-label or label-budget-constrained evaluation changes the picture substantially." ], "competes_with": [ "trajectory-matching", "diffusion-guided-synthesis", "distribution-matching" ], "explanation": null, "arc": [ { "paper": "sre2l-2023", "role": "origin", "note": "Squeeze, recover, relabel: decouples model training from synthetic-data optimization, so resolution and evaluation architecture are free. Reports 60.8% on ImageNet-1K at 50 images per class, with synthesis about 52 times faster than trajectory matching on ConvNet-4." }, { "paper": "cda-2024", "role": "improvement", "note": "Observes that the first recovery iterations fix each image's outline, and schedules augmentation from global to local. Reports 63.2% on ImageNet-1K at 50 IPC and the first distillation of ImageNet-21K at 224×224." }, { "paper": "g-vbsm-2024", "role": "improvement", "note": "Replaces one backbone, one layer set and one statistic with matching across several of each, and reports strong results on small and large datasets with the same method." }, { "paper": "rded-2024", "role": "extension", "note": "Names realism, diversity and efficiency as the requirements at scale and builds synthetic images from selected real content instead of inversion: ImageNet-1K at 10 IPC in 7 minutes on one RTX 4090, reported at 42% top-1 with ResNet-18." }, { "paper": "edc-2024", "role": "improvement", "note": "Maps the design space (soft category-aware matching, learning-rate schedule and other choices) and reports 48.6% on ImageNet-1K with ResNet-18 at 10 IPC." }, { "paper": "dwa-2024", "role": "improvement", "note": "Analyzes diversity among independently synthesized batches and perturbs teacher weights in directed ways so that each batch reflects a different large subset of the data." }, { "paper": "delt-2025", "role": "improvement", "note": "Splits each class's budget into subtasks optimized for different numbers of iterations, breaking the uniformity that shared global supervision imposes on all images of a class." }, { "paper": "fadrm-2025", "role": "improvement", "note": "Adds data-level residual connections that keep local information from real images during pixel optimization, and reports halving training time and peak GPU memory." } ] }, { "id": "generative-latent-optimization", "type": "family", "name": "Optimization in a generative latent space", "paradigm": "generative", "order": 9, "one_line": "Keep a pretrained generator frozen and optimize its latent codes, instead of pixels, with a matching objective.", "core_bet": "Constraining synthetic samples to a generator's manifold regularizes them toward realistic images, which improves generalization to unseen architectures and shrinks the search space.", "formulation": "\\min_{\\{z_i\\}}\\;\\mathcal{L}_{\\text{match}}\\bigl(\\{G(z_i)\\}_{i=1}^{m},\\,\\mathcal{T}\\bigr),\\qquad G\\ \\text{frozen}", "formulation_note": "The matching loss can be any of gradient, trajectory or distribution matching; G has been a GAN, a StyleGAN (at chosen intermediate layers) or a latent diffusion model.", "since": 2022, "status": "active", "strengths": [ "Better cross-architecture generalization than pixel-space optimization with the same objective.", "A lower-dimensional search space and realistic-looking samples." ], "weaknesses": [ "Requires a generator that covers the target distribution; its biases become the dataset's.", "Still pays the full cost of the underlying matching objective, now through the generator." ], "competes_with": [ "parameterization", "diffusion-guided-synthesis" ], "explanation": null, "arc": [ { "paper": "it-gan-2022", "role": "origin", "note": "Freezes a pretrained GAN and learns latent vectors whose images are informative for training rather than realistic, presented as a route to dataset condensation." }, { "paper": "glad-2023", "role": "evidence", "note": "Distills into intermediate feature vectors of a pretrained generator and, added to existing matching objectives, improves cross-architecture generalization in every setting tested." }, { "paper": "h-glad-2025", "role": "improvement", "note": "Optimizes across a hierarchy of GAN layers rather than one fixed latent space, with a class-relevant feature distance to reduce evaluation cost." }, { "paper": "ld3m-2025", "role": "extension", "note": "Backpropagates through a pretrained latent diffusion model to learn distilled latents and class embeddings; a decaying skip connection from the initial noise keeps gradients alive across steps." } ] }, { "id": "generator-as-dataset", "type": "family", "name": "The generator as the distilled artifact", "paradigm": "generative", "order": 10, "one_line": "Distill the dataset into the weights of a (small) generative model, and sample training data from it on demand.", "core_bet": "A model that can produce many informative samples is a more compact and more flexible store of a dataset than any fixed set of images.", "formulation": "\\min_{\\phi}\\;\\mathbb{E}_{z,\\,y}\\;\\mathcal{L}_{\\text{match}}\\bigl(G_{\\phi}(z,y),\\,\\mathcal{T}_y\\bigr),\\qquad \\text{store }G_{\\phi}\\text{ instead of }\\mathcal{S}", "since": 2023, "status": "active", "strengths": [ "Any number of samples can be drawn, so one artifact serves many budgets and training lengths.", "Sample diversity is not capped by a fixed image count." ], "weaknesses": [ "Storage and compute budgets are hard to compare with image-based sets.", "Samples must be generated during training, which shifts cost to the consumer." ], "competes_with": [ "parameterization", "generative-latent-optimization" ], "explanation": null, "arc": [ { "paper": "dim-2023", "role": "origin", "note": "Stores the dataset in a generator trained so that a pool of models gives matching logits on real and generated images; one generator then serves any distillation ratio and larger architectures." } ] }, { "id": "diffusion-guided-synthesis", "type": "family", "name": "Diffusion-based synthesis", "paradigm": "generative", "order": 11, "one_line": "Generate the synthetic set with a pretrained (optionally fine-tuned) diffusion model, steering sampling toward samples that are both representative of the dataset and diverse.", "core_bet": "A large pretrained diffusion model already captures the data distribution; the condensation problem reduces to choosing where in that distribution to spend a small sample budget.", "formulation": "\\hat{\\epsilon}(x_t,t)=\\epsilon_{\\theta}(x_t,t,c_k)\\;-\\;\\lambda_t\\,\\nabla_{x_t}\\,\\mathcal{G}\\bigl(\\hat{x}_0(x_t)\\bigr),\\qquad \\mathcal{S}=\\bigl\\{x_0^{(k)}\\bigr\\}_{k=1}^{K}", "formulation_note": "A generic guided sampler. Members differ in the condition c_k (a class label, a latent prototype or mode, a text prototype, an image–text mode), in the guidance G (representativeness, diversity, influence, learnability, manifold or set-level terms), and in whether the diffusion model is fine-tuned for the purpose.", "since": 2024, "status": "ascendant", "branches": [ { "id": "prototype-mode", "name": "Prototypes and modes", "description": "Find dense regions (clusters, modes) of the data in a latent or embedding space and generate one sample per region.", "papers": [ "d4m-2024", "mgd3-2025", "coda-2026" ] }, { "id": "fine-tuned-prior", "name": "Fine-tuned diffusion priors", "description": "Fine-tune the diffusion model with representativeness and diversity objectives, or with cross-modal contrastive objectives.", "papers": [ "minimax-2024", "ims3-2026" ] }, { "id": "training-free-guidance", "name": "Training-free guidance", "description": "Leave the model untouched and add guidance terms at sampling time toward informative, learnable or diverse samples.", "papers": [ "manifoldgd-2026", "mgd3-2025" ] }, { "id": "inversion-latent-statistics", "name": "Inversion and latent statistics", "description": "Invert real images into the diffusion latent space and select or match latents by their statistics.", "papers": [ "d3hr-2025", "cao2-2025" ] }, { "id": "vision-language-conditioned", "name": "Vision–language conditioning", "description": "Use text prototypes, captions or text-to-image models to condition generation with richer semantics than class names.", "papers": [ "vlcp-2025", "coda-2026" ] } ], "strengths": [ "Cheap relative to matching-based synthesis, often training-free, and it produces realistic high-resolution images.", "Scales to ImageNet-1K and is flexible in the budget, since samples are generated, not optimized." ], "weaknesses": [ "Samples are not optimized for the learner, so the evaluation usually leans on soft labels from a pretrained teacher as well.", "Quality is bounded by the pretrained model's domain; datasets outside it suffer.", "Mode-seeking sampling can reduce within-class diversity, which several members exist to correct." ], "competes_with": [ "decoupled-synthesis", "generative-latent-optimization", "autoregressive-flow-synthesis" ], "explanation": null, "arc": [ { "paper": "minimax-2024", "role": "origin", "note": "Fine-tunes a diffusion model with minimax criteria for representativeness and diversity; reports better accuracy at 100 IPC on ImageWoof with under a twentieth of earlier methods' distillation time." }, { "paper": "d4m-2024", "role": "origin", "note": "Builds the distilled set from class prototypes in a latent diffusion model's space, independent of any matching architecture, to target cross-architecture generalization." }, { "paper": "d3hr-2025", "role": "improvement", "note": "Diagnoses inaccurate distribution matching, noise-induced deviation and separate sampling in earlier diffusion methods; maps dataset latents to a Gaussian domain by DDIM inversion and samples representative latents there." }, { "paper": "mgd3-2025", "role": "improvement", "note": "Drops fine-tuning with distillation losses: discovers modes, guides sampling toward distinct modes to raise intra-class diversity, and stops guidance early to avoid artifacts." }, { "paper": "cao2-2025", "role": "improvement", "note": "Names objective inconsistency and condition inconsistency between diffusion-based distillation and evaluation; selects samples by probability, then refines their latents for conditional likelihood." }, { "paper": "vlcp-2025", "role": "extension", "note": "Adds text prototypes, built from descriptions generated by an open-source language model, to image prototypes when conditioning synthesis." }, { "paper": "coda-2026", "role": "challenge", "note": "Points out that most diffusion-based methods need a diffusion model trained on the full target dataset, and instead steers an off-the-shelf text-to-image model toward a density-discovered core distribution; reports 60.4% on ImageNet-1K at 50 IPC." }, { "paper": "manifoldgd-2026", "role": "improvement", "note": "Training-free guidance that projects the mode-alignment direction onto a local tangent space of the latent manifold at every denoising step, using hierarchical clustering of VAE latents." }, { "paper": "ims3-2026", "role": "challenge", "note": "Argues that diffusion models serve likelihood, not discriminative utility, and over-concentrate in high-density regions; inversion-guided fine-tuning widens coverage and subgroup sampling improves class separability." } ] }, { "id": "autoregressive-flow-synthesis", "type": "family", "name": "Autoregressive and flow-based synthesis", "paradigm": "generative", "order": 12, "one_line": "The generative-prior idea carried to visual autoregressive (next-scale) models and flow-matching generators.", "core_bet": "The structure of other generative families — coarse-to-fine token scales, straight transport paths — offers control points for representativeness that diffusion sampling does not.", "since": 2026, "status": "ascendant", "strengths": [ "Coarse-to-fine or path-level control gives a natural place to inject dataset-level guidance." ], "weaknesses": [ "Very recent; few papers, and evidence is concentrated on ImageNet-style benchmarks." ], "competes_with": [ "diffusion-guided-synthesis" ], "explanation": null, "arc": [ { "paper": "path-guided-fm-2026", "role": "origin", "note": "Presented as the first flow-matching framework for generative distillation: class-conditional transport in a frozen VAE's latent space, with ODE-consistent guidance that lands trajectories on assigned prototypes in few steps." }, { "paper": "hieramp-2026", "role": "origin", "note": "Uses a visual autoregressive model's coarse-to-fine scales, injecting class tokens at each scale to amplify discriminative structure rather than optimizing global proximity." } ] }, { "id": "parameterization", "type": "family", "name": "Synthetic-set parameterization", "paradigm": "orthogonal", "order": 13, "one_line": "Change what is stored — codes and decoders, bases and hallucinators, frequency coefficients, sparse codes, neural fields — so that a fixed storage budget decodes into more or better training samples.", "core_bet": "Pixels are a wasteful storage format for a distilled dataset; a structured representation holds more useful training signal per byte.", "formulation": "\\mathcal{S}=\\bigl\\{g_{\\phi}(z_i)\\bigr\\}_{i=1}^{n},\\qquad \\text{bits}(\\phi)+\\sum_{i}\\text{bits}(z_i)\\;\\le\\;B", "formulation_note": "g is a decoder (identity, upsampling, a factorized combination, an inverse frequency transform, a neural field); it is used with any matching objective.", "since": 2022, "status": "active", "branches": [ { "id": "factorization", "name": "Factorization and sharing", "description": "Bases combined by hallucinator networks, addressable memories, and hierarchical or shared features.", "papers": [ "rtp-2022", "haba-2022" ] }, { "id": "compressed-domains", "name": "Compressed domains", "description": "Down-sampled multi-formation images, frequency coefficients, color palettes, sparse dictionaries, Gaussians, neural fields.", "papers": [ "idc-2022", "fred-2023", "ddif-2025" ] } ], "strengths": [ "Substantially more decoded samples per storage unit, with gains that combine with any objective." ], "weaknesses": [ "Budget accounting differs between papers (images, parameters, bits), which complicates comparison.", "Decoded samples from shared components can be correlated, limiting real diversity." ], "competes_with": [ "generative-latent-optimization", "generator-as-dataset" ], "explanation": null, "arc": [ { "paper": "idc-2022", "role": "origin", "note": "Generates several synthetic images from a limited storage budget by exploiting data regularity, and improves the gradient-matching optimization it is paired with." }, { "paper": "rtp-2022", "role": "improvement", "note": "Learns bases shared across classes and combined by learned addressing functions, so storage need not grow linearly with the number of classes." }, { "paper": "haba-2022", "role": "improvement", "note": "Factorizes a dataset into bases and hallucination networks whose combinations reconstruct images; plug-and-play with existing objectives, with higher cross-architecture accuracy reported." }, { "paper": "fred-2023", "role": "improvement", "note": "Optimizes a selected subset of frequency coefficients per image, chosen by explained variance, on top of existing distillation methods." }, { "paper": "ddif-2025", "role": "extension", "note": "Stores each instance as a neural field over coordinates, which also extends the parameterization to video, audio and 3D voxels." } ] }, { "id": "label-distillation", "type": "family", "name": "Label distillation and soft labels", "paradigm": "orthogonal", "order": 14, "one_line": "Treat labels as part of what is distilled — learned labels, soft teacher labels, relabeling at every epoch — and account for what that costs.", "core_bet": "A label vector can carry far more information about the dataset than a one-hot class index, and this information is cheap relative to images.", "formulation": "\\mathcal{L}_{\\text{student}}=\\mathbb{E}_{x\\in\\mathcal{S},\\,A}\\;\\mathrm{KL}\\Bigl(\\sigma\\bigl(f_{\\text{teacher}}(A(x))/\\tau\\bigr)\\,\\Big\\|\\,\\sigma\\bigl(f_{\\text{student}}(A(x))/\\tau\\bigr)\\Bigr)", "formulation_note": "Relabeling with a pretrained teacher under random augmentation A; earlier work instead learns a fixed label vector jointly with the images.", "since": 2020, "status": "contested", "branches": [ { "id": "learned-labels", "name": "Learned labels", "description": "Optimize label vectors jointly with (or instead of) the synthetic inputs.", "papers": [ "label-distillation-2020" ] }, { "id": "teacher-relabeling", "name": "Teacher relabeling and its cost", "description": "Soft labels from a pretrained teacher, and methods that prune, quantize or lighten the label storage they require.", "papers": [ "lpld-2024", "helio-2025", "gift-2025" ] }, { "id": "critiques", "name": "Critiques", "description": "Analyses showing how much of the reported accuracy the labels, rather than the images, account for.", "papers": [ "label-worth-2024" ] } ], "strengths": [ "Large accuracy gains in essentially every paradigm, particularly at ImageNet scale." ], "weaknesses": [ "Per-epoch soft labels for many augmented crops can require more storage than the images they label.", "Makes comparisons with hard-label methods invalid unless reported separately; a teacher trained on the full data leaks its knowledge into the student." ], "competes_with": [], "explanation": null, "arc": [ { "paper": "label-distillation-2020", "role": "origin", "note": "Distills labels for a small set of real images instead of distilling images, and reports better compatibility with off-the-shelf optimizers and architectures than image-based distillation." }, { "paper": "label-worth-2024", "role": "challenge", "note": "Ablations find that soft labels, not the image-synthesis technique, are the main factor behind state-of-the-art accuracy, and that only soft labels carrying structured information help." }, { "paper": "lpld-2024", "role": "improvement", "note": "Batching synthesis within classes raises within-class diversity so that soft labels can be pruned at random: ImageNet-1K soft labels at 200 IPC shrink from 113 GB to 2.8 GB." }, { "paper": "helio-2025", "role": "improvement", "note": "Replaces stored soft labels with a CLIP-based image-to-label projector adapted by low-rank fine-tuning, at about 0.003% of the soft-label storage." }, { "paper": "gift-2025", "role": "improvement", "note": "Finds students highly sensitive to the loss applied to soft labels and adds label refinement with a cosine-similarity loss, notably improving generalization across optimizers." } ] }, { "id": "optimization-recipes", "type": "family", "name": "Optimization and training recipes", "paradigm": "orthogonal", "order": 15, "one_line": "Improvements to how any condensation objective is optimized — initialization, model augmentation, sample weighting, curricula, multi-size and prunable sets.", "core_bet": "Much of the gap between a condensation objective and its potential lies in optimization details that transfer across families.", "since": 2023, "status": "active", "strengths": [ "Often plug-in improvements that apply to several families at once." ], "weaknesses": [ "Gains are reported against different baselines and budgets and are hard to attribute." ], "competes_with": [], "explanation": null } ], "problems": [ { "id": "accuracy-does-not-isolate-data-quality", "type": "problem", "statement": "Reported accuracy does not isolate the quality of the synthetic data.", "why": "Modern evaluation pipelines add teacher soft labels, strong augmentation and long schedules. Ablations and re-evaluations show that these additions explain much of the reported accuracy and of the differences between methods, so a better number does not reliably mean a better condensed dataset.", "status": "partially-addressed", "settings": [ "image-classification" ], "evidence": [ { "paper": "label-worth-2024", "note": "Ablations indicate soft labels, not the image-generation technique, are the main factor behind state-of-the-art performance." }, { "paper": "dd-ranking-2025", "note": "Improvements often stem from additional evaluation techniques; even randomly sampled images can do better under them." }, { "paper": "rd3-2026", "note": "Much of the performance variation among decoupled methods is attributable to inconsistent post-evaluation protocols." }, { "paper": "hard-truths-soft-labels-2026", "note": "With abundant soft labels and teacher distillation, performance saturates regardless of subset size or quality." } ], "attacked_by": [ "dc-bench-2022", "dd-ranking-2025", "rd3-2026" ] }, { "id": "condensation-amplifies-unfairness", "type": "problem", "statement": "Condensation tends to preserve the majority and amplify unfairness toward minority groups.", "why": "Matching objectives align the synthetic set with the whole distribution, which the majority dominates. With only a handful of samples per class, minority subgroups can disappear from the distilled data, and models trained on it inherit the gap, for images and graphs alike.", "status": "partially-addressed", "settings": [ "image-classification", "graph" ], "evidence": [ { "paper": "fairdd-2025", "note": "Distillation does not fix and typically worsens unfairness toward protected-attribute minority groups, because matching aligns to majority-dominated distributions." }, { "paper": "fair-graph-distillation-2023", "note": "GNNs trained on condensed graphs can show more severe group-fairness problems than GNNs trained on the original graph." }, { "paper": "fair-dataset-distillation-via-2026", "note": "Fairness gaps come from mismatched predictive patterns between subgroups, not only group-size imbalance, and persist after correcting for imbalance." }, { "paper": "a-comprehensive-study-on-2023", "note": "Reports that distillation can amplify unfairness across classes." } ], "attacked_by": [ "fairdd-2025", "fair-graph-distillation-2023", "fair-dataset-distillation-via-2026" ] }, { "id": "distilled-data-outside-the-protocol", "type": "problem", "statement": "Distilled data works inside the standard evaluation protocol, but does not substitute for real data outside it.", "why": "If a distilled set mainly encodes early training dynamics of one training recipe, it will not serve the uses condensation is promoted for (continual learning, architecture search, pre-training) whenever those change the recipe.", "status": "open", "settings": [ "image-classification" ], "evidence": [ { "paper": "what-is-dd-learning-2024", "note": "Distilled data cannot substitute for real data outside the standard evaluation setting, and it retains performance by compressing information about early training dynamics." } ], "attacked_by": [] }, { "id": "distilled-sets-versus-coresets", "type": "problem", "statement": "At large scale, it is not established that synthesized sets beat well-chosen real subsets.", "why": "The case for synthesis is that a subset of real images cannot express as much as optimized data. Under hard labels and standardized protocols on ImageNet-scale benchmarks, recent re-evaluations find coresets comparable or better at far lower construction cost.", "status": "contested", "settings": [ "image-classification" ], "evidence": [ { "paper": "dd-vs-coresets-2026", "note": "State-of-the-art distilled sets are comparable to or worse than coresets on ImageNet-1K, ImageNet-100 and ImageNette under standardized protocols, and cost substantially more." }, { "paper": "hard-truths-soft-labels-2026", "note": "With hard labels, only RDED reliably beats random selection on ImageNet-1K among the methods tested, and it can still trail strong coresets." } ], "attacked_by": [ "hard-truths-soft-labels-2026" ] }, { "id": "graph-condensation-evaluation-is-misleading", "type": "problem", "statement": "Graph condensation needs full-graph training to produce a small graph, and its standard metrics hide the true cost.", "why": "The dominant gradient- and trajectory-matching graph methods train on the full graph to build the condensed one, so the efficiency they report excludes the most expensive step. Node compression ratios ignore storage of features and structure and the cost of condensation itself, and benchmark protocols vary. This mirrors the evaluation reckoning in image condensation.", "status": "open", "settings": [ "graph" ], "evidence": [ { "paper": "graph-condensation-reset-2026", "note": "Argues gradient-matching condensation requires full-dataset training, generalizes poorly across GNN architectures, and relies on misleading metrics such as node compression ratio." }, { "paper": "gc4nc-2024", "note": "A 12-method benchmark that documents inconsistent evaluation practices across graph condensation papers." }, { "paper": "gcond-2022", "note": "The founding method imitates GNN training on the original graph via gradient matching, the design the position paper critiques." } ], "attacked_by": [] }, { "id": "label-storage-is-uncounted", "type": "problem", "statement": "The budget counts images, but not the labels and auxiliary structures that make them work.", "why": "Images-per-class ignores soft labels, decoders, bases and generators. At ImageNet scale the stored soft labels can exceed the images many times over, so two methods with the same IPC can differ by orders of magnitude in what they actually store.", "status": "partially-addressed", "settings": [ "image-classification", "vision-language" ], "evidence": [ { "paper": "lpld-2024", "note": "Soft-label storage in ImageNet condensation exceeds the condensed images by over 30×; compressed from 113 GB to 2.8 GB at 200 IPC." }, { "paper": "helio-2025", "note": "State-of-the-art methods rely on soft labels whose storage can approach that of the original dataset; replaces them with an image-to-label projector." } ], "attacked_by": [ "lpld-2024", "helio-2025" ] }, { "id": "multimodal-condensation-has-not-scaled", "type": "problem", "statement": "Multimodal condensation has not scaled beyond about a thousand image–text pairs, and recall stays far below full-data training.", "why": "Image classification reached ImageNet-1K and 21K once synthesis was decoupled from the learner. Image–text condensation is still evaluated on Flickr30K and COCO at 100–1,000 pairs, where retrieval recall remains a fraction of what full-data training achieves, and the one attempt at a larger corpus sits near the random floor. The setting has not had its scale-breaking moment.", "status": "open", "settings": [ "vision-language", "audio-visual-omni" ], "evidence": [ { "paper": "lors-2024", "note": "Text→image R@1 of 10.0 on Flickr30K at 500 pairs, under a protocol whose full-data retrieval is several times higher." }, { "paper": "mtt-vl-2024", "note": "Reports a full-data NFNet+BERT ceiling of image→text R@1 33.9 and text→image R@1 27.3 on Flickr30K, against 13.3 and 7.9 for its distilled set at 1,000 pairs." }, { "paper": "edge-2025", "note": "On CC3M at 1,000 pairs its retrieval R@1 is 0.1–0.2, near the random floor." }, { "paper": "hopa-2026", "note": "The omnimodal extension is demonstrated for exactly three modalities; scaling with modality count is argued, not shown." } ], "attacked_by": [ "covmatch-2025", "mdm-2026", "edge-2025", "pds-2026" ] }, { "id": "multimodal-protocols-are-incompatible", "type": "problem", "statement": "Vision–language condensation results are reported under incompatible evaluation protocols.", "why": "Image–text methods differ in which encoders are trained during distillation and evaluation, and in which text encoder is used at all. At least three recipes are in use (frozen BERT, trainable BERT, CLIP text tower), and re-runs of the same baseline at the same budget disagree between papers. Without a shared, versioned benchmark, state-of-the-art claims cannot accumulate across papers as they do for image classification.", "status": "open", "settings": [ "vision-language" ], "evidence": [ { "paper": "covmatch-2025", "note": "Makes the BERT text encoder trainable; its ablation shows freezing BERT alone drops Flickr30K mean recall at 500 pairs from 38.4 to 29.4, so the protocol choice is worth more than many method differences." }, { "paper": "pds-2026", "note": "Evaluates with a frozen CLIP text encoder and non-CLIP image backbones, re-implementing LoRS and TESLA-VL with a CLIP text tower; its numbers cannot be pooled with BERT-based tables." }, { "paper": "ptm-st-2026", "note": "Its re-run of LoRS at 500 pairs (text→image R@1 12.7) differs from LoRS's published 10.0, which RepBlend's re-run reproduces exactly; no paper documents the cause." }, { "paper": "edge-2025", "note": "Evaluates with a pretrained NFNet and a frozen BERT-base, the protocol shared with LoRS and RepBlend but not with CovMatch or RAHA." } ], "attacked_by": [] }, { "id": "privacy-benefit-is-disputed", "type": "problem", "statement": "Whether condensed data offers any privacy protection is disputed.", "why": "Condensation is often promoted as a way to share data safely, because synthetic images are not real records. The central claim of a privacy benefit and its rebuttal were published the same year and point in opposite directions, and later benchmarking finds leakage reduced but not removed. Without formal guarantees, a distilled dataset should not be treated as private.", "status": "contested", "settings": [ "image-classification" ], "evidence": [ { "paper": "privacy-for-free-2022", "note": "Argues condensation acts as a private-data generator for free, with a bound on any one raw sample's influence for linear feature extractors, supported by membership-inference experiments." }, { "paper": "no-free-lunch-privacy-2022", "note": "Argues that evaluation and analysis are flawed and do not show a statistically significant privacy benefit; DP-SGD gives both better accuracy and provably lower membership-attack success." }, { "paper": "a-comprehensive-study-on-2023", "note": "Reports that distillation reduces but does not eliminate privacy leakage under membership inference." }, { "paper": "soft-label-leakage-2026", "note": "Teacher soft labels can transfer memorized information about held-out examples to a student." } ], "attacked_by": [] }, { "id": "soft-labels-transfer-teacher-knowledge", "type": "problem", "statement": "Soft labels transfer teacher knowledge that is not in the distilled data at all.", "why": "If a student can recover what a teacher memorized about held-out examples from soft labels alone, then a teacher-relabeled distilled set is partly a compressed teacher. That changes both what the accuracy means and what privacy a distilled dataset can offer.", "status": "open", "settings": [ "image-classification" ], "evidence": [ { "paper": "soft-label-leakage-2026", "note": "Students trained on teacher soft labels reach non-trivial accuracy on held-out memorized data they never observed, sometimes perfect accuracy." }, { "paper": "label-worth-2024", "note": "Soft labels must contain structured information to be beneficial." } ], "attacked_by": [] } ], "papers": [ { "id": "self-supervised-representation-guided-2026", "type": "paper", "title": "Self-Supervised Representation-Guided Generative Dataset Distillation", "short": "SRG", "arxiv": "2608.03218", "url": null, "date": "2026-08-04", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Mingzhuo Li", "Guang Li", "Linfeng Ye", "Jiafeng Mao", "Takahiro Ogawa", "Konstantinos N. Plataniotis", "Miki Haseyama" ], "kind": "method", "settings": [ "self-supervised" ], "families": [ "diffusion-guided-synthesis" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Targets distillation for downstream training with frozen pretrained self-supervised encoders and lightweight adapters rather than randomly initialized networks: builds class-wise prototypes from real-image SSL representations and translates three SSL-space objectives (prototype alignment, inter-class discrimination, intra-class assignment) into diffusion guidance, anchoring early denoising to the latent of the nearest real image and later denoising to the SSL objectives. Reports consistent gains over evaluated generative baselines across datasets and IPC settings, with transfer across different pretrained representation spaces.", "abstract": "Dataset distillation compresses a large training set into a compact synthetic set while retaining its downstream utility. Most existing methods target randomly initialized networks, whereas modern vision systems often adapt frozen pretrained encoders with lightweight modules. Distilled samples should therefore preserve the discriminative geometry of the pretrained representation space, which existing generative objectives do not explicitly consider. We propose self-supervised representation-guided generative dataset distillation (SRG), a framework that translates the SSL geometry into diffusion guidance. Specifically, SRG constructs class-wise prototypes from real-image SSL representations and performs guidance through three SSL-space objectives for prototype alignment, inter-class discrimination, and intra-class assignment. During diffusion sampling, it adopts a stage-wise guidance strategy: early denoising is anchored to the latent of the real image whose SSL representation is nearest to the assigned prototype, whereas later denoising is guided by the SSL-space objectives. This division preserves the visual realism provided by the generative prior while progressively steering samples toward representative and class-discriminative regions of the SSL representation space. SRG consistently outperforms the evaluated generative baselines across multiple datasets and IPC settings. A cross-encoder evaluation further indicates transfer across pretrained representation spaces. These results demonstrate the effectiveness of representation-guided generation for dataset distillation with pretrained SSL models.", "notes": "" }, { "id": "efficient-video-dataset-distillation-2026", "type": "paper", "title": "Efficient Video Dataset Distillation via Cluster-Guided Prototype Blending", "short": "ProtoBlend", "arxiv": "2608.03269", "url": null, "date": "2026-08-04", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Chongle Ren", "Guang Li", "Wenbo Huang", "Naoki Saito", "Takahiro Ogawa", "Miki Haseyama" ], "kind": "method", "settings": [ "video" ], "families": [ "dataset-quantization", "decoupled-synthesis" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Builds distilled video sets without any gradient-based optimization of the stored clips: a teacher model selects one high-confidence temporal segment per source video, cluster-guided allocation partitions the selected clips per class in the teacher's feature space and assigns one distilled slot per cluster, and each slot is blended with an in-cluster anchor clip while their teacher predictions are mixed with the same coefficient. Reports a competitive accuracy-efficiency trade-off against iteratively optimized video distillation methods on four trimmed action-recognition benchmarks.", "abstract": "Video dataset distillation aims to compress a large video dataset into a compact surrogate set that preserves its training utility. Most existing approaches synthesize condensed videos through iterative optimization, whose cost is amplified by the temporal dimension. Rather than further reducing the number of optimized variables, we investigate whether effective distilled videos can be constructed without gradient-based optimization of the stored videos. Such a construction-based approach must address three challenges: selecting informative temporal segments, covering diverse intra-class variations under a limited videos-per-class budget, and increasing the information carried by each stored sample. To this end, we propose ProtoBlend, an efficient select-allocate-blend framework. First, teacher-guided temporal clip selection retains a high-confidence segment from each source video. Second, cluster-guided prototype allocation partitions the selected clips in the teacher feature space and assigns one distilled slot to each intra-class cluster. Third, each prototype is blended with an in-cluster anchor, while their teacher predictions are combined using the same coefficient to provide mixture-source supervision. Experiments on four trimmed action-recognition benchmarks demonstrate that ProtoBlend achieves a competitive accuracy-efficiency trade-off without iterative optimization of the distilled videos.", "notes": "" }, { "id": "infmatch-2026", "type": "paper", "title": "Dataset Distillation by Influence Matching", "short": "InfMatch", "arxiv": "2607.16859", "url": null, "date": "2026-07-18", "venue": { "name": "CVPR 2026", "status": "conference" }, "authors": [ "Haoru Tan", "Wang Wang", "Sitong Wu", "Xiuzhe Wu", "Yangtian Sun", "Chirui Chang", "Shaofeng Zhang", "Xiaojuan Qi" ], "kind": "method", "settings": [ "image-classification", "vision-language" ], "families": [ "meta-learning-bptt", "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/hrtan/infmatch", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Replaces process-level surrogates (per-step gradients or training trajectories) with a differentiable, sample-level estimator of each synthetic point's influence on the converged model parameters, computed in linear time by unrolling the optimization dynamics with a first-order Taylor approximation, then learns the synthetic set by matching this influence to that of the real data; reports 31.5% accuracy on Tiny-ImageNet at IPC-10 (a 4.7-point gain over NCFM) and extends to vision-language distillation on Flickr30K.", "abstract": "We revisit dataset distillation from an outcome-centric perspective. Rather than aligning process surrogates (per-step gradients or training trajectories), Influence Matching (Inf-Match) aligns the final outcome of training: it learns a compact synthetic set whose effect on the converged parameters matches that of the full dataset. Concretely, we introduce a fully differentiable, sample-level influence estimator that quantifies parameter shifts from adding or removing data, without time-consuming inverse-Hessian products or convexity assumptions. The estimator runs in linear time by unrolling the optimization dynamics and applying a first-order Taylor approximation. We then learn the synthetic set by minimizing the mismatch between its influence and that of the real dataset, yielding outcome alignment rather than heuristic process imitation. Inf-Match delivers the best accuracy across standard classification benchmarks. For instance, on Tiny-ImageNet (IPC=10), Inf-Match attains 31.5\\%, a +4.7\\% improvement over NCFM. Beyond classification, Inf-Match scales to vision-language distillation on Flickr30K, outperforming strong process-matching baselines. For instance, with 200 to 1000 synthetic samples, our method achieved a leading impressive average on image/text retrieval tasks, higher than NCFM by 2.5\\%. The code will be released via https://github.com/hrtan/infmatch.", "awesome_sections": [ "Main / Better Optimization" ], "notes": "Influence Matching does not cleanly fit any single P1-P2 family; classified under meta-learning-bptt (it unrolls optimization dynamics like BPTT methods) plus optimization-recipes — flagged for senior review as a possible new mechanism.", "builds_on": [ "ncfm-2025" ] }, { "id": "adaptive-latent-trajectory-anchoring-2026", "type": "paper", "title": "Adaptive Latent Trajectory Anchoring for Action Segmentation Dataset Condensation", "arxiv": "2607.09081", "url": null, "date": "2026-07-10", "venue": { "name": "ECCV 2026", "status": "conference" }, "authors": [ "Artheme Gauthier-Villar", "Guodong Ding", "Angela Yao" ], "kind": "method", "settings": [ "video" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "prior": [ "diffusion" ], "parameterization": [ "generative-latent" ] }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Condenses action-segmentation video datasets by replacing VAE-based iterative latent optimization with deterministic DDIM latent trajectories anchored by sparse latent points on the noise manifold, and adaptively reallocates the anchoring budget across segments by per-segment reconstruction difficulty; reports performance parity with full-data training at a 2.4% condensation ratio on the Breakfast dataset.", "abstract": "Dataset condensation for action segmentation synthesizes compact, informative representations of long, untrimmed video datasets. The existing approach relies on Variational Autoencoders and an iterative latent optimization; it is computationally expensive and suffers from over-smoothed reconstructions and rigid temporal constraints. This paper proposes to shift the condensation paradigm from optimization-based inversion to deterministic latent mapping. By leveraging Denoising Diffusion Implicit Models, we represent action segments as continuous trajectories anchored by sparse latent points in the noise manifold. To maximize representational efficiency, we introduce an adaptive allocation mechanism that dynamically redistributes the anchoring budget based on segment-wise reconstruction difficulty. Extensive experiments demonstrate that our framework significantly outperforms state-of-the-art methods in segmentation performance across common datasets. Notably, our approach achieves performance parity with real data training while maintaining a condensation ratio of 2.4\\% on Breakfast dataset.", "awesome_sections": [ "Applications / Video" ], "notes": "" }, { "id": "cim-2026", "type": "paper", "title": "Condensing Large-Scale Datasets Directly with Minimal Information Loss", "short": "CIM", "arxiv": "2607.00916", "url": null, "date": "2026-07-01", "venue": { "name": "ECCV 2026", "status": "conference" }, "authors": [ "Xinyi Shang", "Peng Sun", "Bei Shi", "Zixuan Wang", "Tao Lin" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "facets": { "labels": "soft-relabel", "prior": [ "pretrained-classifier" ], "regime": "decoupled", "parameterization": [ "pixels" ], "scale": [ "imagenet-1k" ] }, "tier": "notable", "links": { "code": "https://github.com/LINs-lab/CIM", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "The squeeze-recover-relabel pipeline (SRe2L, CDA, G-VBSM, EDC, DWA, CV-DD, DELT, NRR-DD) inverts a pretrained classifier's batch-norm statistics and logits to recover pixels, then relabels with the same classifier. RDED instead selects and stitches real crops using a pretrained observer, with no inversion step at all.", "problem": "The paper argues squeeze (data to model) and recover (model back to images) form an implicit dual compression that loses information twice, degrading cross-architecture generalization and costing many optimization iterations. It further shows (Table 7 / App. E) that this loss shifts the distilled images' distribution away from the real data the relabeling classifier was trained on: disabling Relabel collapses SRe2L and G-VBSM to near-chance on ImageNet-1K (1.1% and 0.8% at IPC=10), and applying Relabel post hoc to non-Relabel methods (ADD, DataDAM) helps only in the earliest, least-optimized stage of their distillation before images drift from real-image initialization (Fig. 1) -- evidence, formalized as Proposition 1 (a two-Gaussian distribution-shift bound), that a relabeler trained on real data becomes unreliable once its input distribution shifts.", "idea": "Abandon inversion entirely: select real-image subsets per class (by default with RDED's selection score), then directly minimize a bounded \"effective information gap\" between each real subset and its corresponding synthetic image (RandomCrop views of a real-image-initialized canvas), matching intermediate-layer features of a single pretrained model under multiple transformations rather than matching BN statistics. Because the synthetic images stay distributionally close to real data by construction, the same pretrained model can then relabel them reliably.", "method": "One pretrained observer/teacher network $\\phi_{\\theta_T}$ (e.g. ResNet-18), no student in the loop. Stage 1 selects, per class, the top $N\\times\\text{IPC}$ real images by teacher cross-entropy loss from a candidate subset of size 300 (RDED-style, but CIM is stated to be selection-agnostic; Table 6 shows it also works with Random/K-means/Herding selection). Stage 2 initializes each distilled image by concatenating $N{=}4$ selected real images (resized), then iteratively (M=200 iterations) applies RandomCrop augmentations and minimizes an $\\ell_2$ feature-gap loss (Eq. 9) between a middle-layer representation of the distilled crop and the corresponding real image, bounding the intractable KL \"effective information gap\" (Def. 2, Thm. 4.1) by an $\\ell_2$ feature distance; middle layers are used (not logits) to preserve texture as well as semantics (Fig. 4c ablation). Relabeling generates one soft label per transformed view $\\zeta_k(\\tilde x)$ (Eq. 10), extending SRe2L's single-crop relabel to multiple transformation types. No bi-level coupling to a student; images can be optimized one at a time rather than as a synchronized batch.", "evidence": "ImageNet-1K, ResNet-18, relabel+KD (Table 2): IPC=10 CIM 48.7% vs DELT 45.8%, NRR-DD 46.1%, GIFT 43.2%, RDED 41.1%, WMDD 38.2%, G-VBSM 35.7%, CIM's own re-run of SRe2L 31.1% (note: this SRe2L re-run differs both from SRe2L's own-reported 21.3% and from DELT's re-run of 41.9% at the same cell -- a third, still different value for the same nominal baseline); IPC=50 CIM 60.4% vs NRR-DD 60.1%, WMDD 57.6%, DELT 59.2%; IPC=100 CIM 62.4%, tied with DELT. ResNet-50 (Table 3): IPC=10 CIM 54.3% vs EDC 54.1%, CV-DD 51.3%, RDED 46.2%; IPC=50 CIM 65.9% vs EDC 64.3%. Cross-architecture at IPC=10 (Table 4, ResNet-18-distilled data transferred): CIM leads on all 6 tested architectures including ViT-T/16 (10.8% vs RDED 8.5%, SRe2L 3.2%). Cost: distills ImageNet-1K IPC=10 in 80 minutes on one RTX-4090 (abstract, Conclusion); Table 5 per-100-image generation time on ResNet-18: CIM 25.34s / 1.56GB vs SRe2L 191.14s / 3.62GB and G-VBSM 259.84s / 4.94GB, but RDED remains faster and lighter (2.78s / 0.92GB) as the optimization-free reference. Genuine hard-label result (Table 7, App. E, ResNet-18, IPC=10, no relabel): ImageNet-1K -- CIM 22.0% vs RDED 19.7%, SRe2L 1.1%, G-VBSM 0.8%; CIFAR-10 -- CIM 51.6% vs RDED 35.2%, G-VBSM 32.3%, SRe2L 10.9%. Key ablation: removing Relabel collapses SRe2L/G-VBSM to near-chance on ImageNet-1K (Table 7) while CIM and RDED (both real-content-anchored) degrade far less, isolating distributional proximity to real data, not the relabeling mechanism itself, as what makes Relabel effective.", "limitations": "The paper's own stated limitation (App. A) is that CIM cannot match RDED's optimization-free efficiency despite being much faster than inversion-based methods. The 80-minute headline cost excludes teacher pretraining. CIM still depends on one pretrained observer for both selection-score ranking and the feature-gap objective, so it inherits a dependence on that model's own training recipe (a dependency this family's other papers, e.g. sre2l-2023, already flag as unresolved). Soft-label storage across the multiple transformed views per image is not quantified. Gains over the next-best methods (NRR-DD, DELT) are modest at IPC=50/100 (roughly 0.3-2.8 points) even though the IPC=10 margin is larger, and the hard-label advantage over RDED (Table 7) is smaller than the relabel+KD advantage, suggesting part of CIM's edge still comes from the relabeling step it argues is only reliable once distribution shift is fixed." }, "summary": "Argues that the squeeze-recover-relabel pipeline's implicit dual compression (data to model, model back to images) causes a distribution shift that makes the pretrained model an unreliable relabeler; CIM instead directly quantifies and minimizes a metric-driven information gap between real and synthetic distributions, reporting 48.7% top-1 on ImageNet-1K at IPC=10 in 80 minutes on one RTX-4090, ahead of NRR-DD and DELT by 2.6 and 2.9 points respectively.", "abstract": "Recent advancements in scaling dataset distillation rely heavily on decoupled information extraction pipelines, comprising SQUEEZE, RECOVER, and RELABEL stages. Despite their scalability to large-scale datasets, these methods suffer from prohibitive computational overhead and poor cross-architecture generalization. In this paper, we reveal the root cause of these bottlenecks: the implicit dual-compression process, from data to model and back to images, inherently induces severe information loss. Crucially, we empirically and theoretically demonstrate that this loss creates a distribution shift that fundamentally compromises the widely adopted RELABEL strategy, transforming the pre-trained model into an unreliable labeler that yields sub-optimal labels. To overcome these critical flaws, we propose CIM, a novel, metric-driven framework that abandons the flawed dual-compression paradigm. Instead, CIM explicitly quantifies and minimizes the information gap between the original and synthetic datasets. By directly aligning the data distributions, our approach ensures high-fidelity information condensation and inherently satisfies the prerequisites for effective relabeling. Extensive experiments demonstrate that CIM establishes a new state-of-the-art. Notably, it distills ImageNet-1K at an IPC=10 in merely 80 minutes on a single RTX-4090 GPU, achieving an unprecedented 48.7% Top-1 accuracy on ResNet-18 and significantly outperforming previous SOTA approaches, such as NRR-DD and DELT, by 2.6% and 2.9%, respectively. Our code is available at https://github.com/LINs-lab/CIM.", "builds_on": [ "nrr-dd-2025", "delt-2025" ], "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "raha-2026", "type": "paper", "title": "Rank-Aware Hyperbolic Alignment for Vision-Language Dataset Distillation", "short": "RAHA", "arxiv": "2606.29464", "url": null, "date": "2026-06-28", "venue": { "name": "ECCV 2026", "status": "conference" }, "authors": [ "Jongoh Jeong", "Sun-Kyung Lee", "Kuk-Jin Yoon" ], "kind": "method", "settings": [ "vision-language" ], "families": [ "distribution-matching" ], "facets": { "labels": "hard", "prior": [ "pretrained-classifier" ], "regime": "single-level", "parameterization": [ "pixels", "embedding-features" ], "scale": [ "flickr30k-coco" ] }, "tier": "notable", "links": { "code": "https://github.com/andyj1/raha", "project": "https://andyj1.github.io/raha/" }, "status": { "verified": true, "verified_on": "2026-09-14", "source": "arxiv-api", "depth": "full-text" }, "summary": "RAHA argues that full-dimensional Euclidean alignment is too restrictive for the rank-deficient image-text correlation typical of vision-language distillation (shared semantics concentrate in a low-dimensional range, with the remainder in a weakly correlated residual subspace), and instead lifts representations to hyperbolic space, optimizing distilled pairs with asymmetric objectives that enforce geodesic alignment on the shared range while regularizing the residual subspace to keep modality-private diversity. The paper reports competitive cross-modal retrieval and improved transfer robustness under fixed budgets versus prior similarity-level low-rank alignment (LoRS).", "abstract": "Vision-language dataset distillation (VLDD) compresses a large image-text paired dataset into a small set of synthetic pairs that can efficiently train contrastive vision-language models under strict data and compute budgets. Most existing methods match expert trajectories or cross-modal statistics, yet still enforce full-dimensional alignment in a Euclidean embedding space. This is often overly restrictive due to rank-deficient image--text correlation, with shared semantics concentrated in a low-dimensional range and remaining variation spread across a weakly correlated residual subspace. LoRS relaxes alignment at the similarity level by low-rank factorization, but does not explicitly control dominant alignment capacity and structure in the representation space. We thus propose a rank-aware hyperbolic alignment (RAHA) that combines hierarchical geometry with explicit alignment-capacity control. RAHA lifts multimodal representations to hyperbolic space and optimizes distilled pairs with asymmetric objectives that enforce geodesic alignment in the shared range while regularizing the residual subspace to preserve modality-private diversity and improve transfer robustness. Experiments on benchmarks show that RAHA demonstrates competitive cross-modal retrieval and improved transfer indicators under fixed budgets.", "builds_on": [ "lors-2024", "covmatch-2025" ], "explained": { "before": "LoRS relaxes the rigid one-to-one pairing assumption at the *similarity-label* level via low-rank factorization of a similarity matrix, but still aligns image and text representations with full-dimensional Euclidean objectives. CovMatch matches cross-covariance between real and synthetic features uniformly across all feature directions, with a trainable text encoder, but does not distinguish which directions actually carry shared cross-modal information from which do not.", "problem": "The paper diagnoses that real image-text correlation is rank-deficient: shared semantics concentrate in a low-dimensional \"range\" (the top directions of the image-text cross-covariance, found via SVD), while the remaining, larger residual subspace carries mostly weakly-correlated, modality-private variation. Enforcing uniform full-dimensional Euclidean alignment (as LoRS and CovMatch do) either wastes alignment capacity on the noisy residual directions or, when capacity is scarce under heavy compression (as few as 100 pairs), suppresses complementary modality-private information there entirely. Euclidean geometry also has no natural inductive bias for the hierarchical (coarse-scene-to-fine-attribute) structure of image-text semantics.", "idea": "Lift image and text representations to hyperbolic space, whose geometry naturally accommodates hierarchical semantic structure, and split the alignment objective by rank: explicitly identify a low-dimensional \"range\" subspace via SVD of the real cross-covariance, enforce strong geodesic alignment only within that shared range, and separately regularize (rather than force-align) the orthogonal residual subspace so it can retain modality-private diversity instead of being over-compressed toward cross-modal agreement.", "method": "Synthetic image pixels and continuous text-token embeddings (with an attention mask, no discrete tokens) are optimized; a pretrained NFNet image encoder and BERT text encoder with projection heads are held frozen throughout distillation (only the synthetic parameters are trained), following an alternating procedure (Algorithm 1: 50 synthetic-update steps per 1 real-data model-update step, up to 200 iterations, real-batch features computed under stop-gradient) similar in spirit to CovMatch's fixed-per-step scheme. Projected features are lifted onto a Lorentz-model hyperboloid via an exponential map with a learnable curvature-scale factor $s$ (Eq. 3-4); pairwise geodesic distance is $d^c(u,v)=\\frac1{\\sqrt c}\\,\\text{arcosh}(-c\\langle u,v\\rangle_{\\mathcal L})$ (Eq. 5), and a hyperbolic InfoNCE contrastive loss on synthetic pairs uses $-d^c(h^v_i,h^t_j)/\\tau$ as logits (Eq. 6-7). Rank-aware capacity control (Eq. 9-11): the real batch's tangent-space cross-covariance $C_{real}$ is SVD-factorized ($C_{real}=U\\Sigma V^\\top$), and an adaptive rank $k$ is chosen as the smallest $k'$ whose top-$k'$ singular-value energy covers a threshold fraction $\\rho=0.95$ of total energy, splitting features into a $k$-dimensional range and an orthogonal-complement residual. The range loss (Eq. 21) combines Sinkhorn-matched relevance distributions with an energy regularizer penalizing synthetic range coupling falling below the real range coupling; the residual loss (Eq. 24) matches residual relevance distributions but adds a compression regularizer penalizing residual energy exceeding range energy. The total objective (Eq. 25) is $\\mathcal L_{total}=\\mathcal L_{hitc}+\\lambda_{range}\\mathcal L_{range}+\\lambda_{residual}\\mathcal L_{residual}$ ($\\lambda_{range}=0.8$, $\\lambda_{residual}=0.4$).", "evidence": "NFNet+BERT protocol; Table 4 reports averaged IR and TR (mean over K=1,5,10), not per-K breakdowns. Flickr30K@100: RAHA IR/TR/Mean 18.7/22.7/20.7, vs their reproduction of CovMatch 20.9/24.6/22.8 - RAHA trails at this smallest budget. @200: RAHA 23.5/27.9/25.7 vs CovMatch 20.2/23.7/22.0 - RAHA ahead. @500: RAHA 30.0/35.9/32.9 vs CovMatch 26.3/31.5/28.9 - RAHA ahead by a larger margin. COCO@100: RAHA 6.6/7.7/7.2 vs CovMatch 6.5/7.4/7.0 (near tie). @200: RAHA 9.3/11.0/10.2 vs CovMatch 7.4/9.2/8.3. @500: RAHA 12.6/14.9/13.7 vs CovMatch's own table row 9.9/8.3/11.2 (the paper's own printed mean of 11.2 does not arithmetically match the average of 9.9 and 8.3, an inconsistency in the source table that could not be resolved from the text). At 1000 pairs (their appendix Table A6): Flickr30K mean RAHA 38.0 vs CovMatch 28.4; COCO mean RAHA 18.6 vs CovMatch 14.2. A reproduction note (their Appendix Table A4) states published CovMatch numbers often exceed the authors' own reproduction by 7-12 points, and RAHA is reported as outperforming CovMatch under the reproduced protocol at 200/500 pairs. Cross-architecture transfer (Table 5, Flickr8K, mean recall on NF-ResNet/NF-RegNet/ViT-B): RAHA improves from 9.9 (N=100) to 12.7 (N=500) while CovMatch stays roughly flat (9.6 to 8.7). Cost: the paper states that at batch size 1 RAHA and CovMatch are similar (~25s/iteration), but the gap grows with batch size due to the SVD/lifting overhead - at batch size 64, RAHA takes ~400s versus CovMatch's ~55s per iteration, i.e. RAHA is substantially slower at scale. The key ablation (Figure 2, Flickr8K N=100) is additive: hyperbolic InfoNCE alone gives ~18 mean recall; adding range matching raises it to ~19; adding residual regularization on top reaches ~20 - both the range and residual terms contribute, and residual matching without range anchoring is reported as weak on its own.", "limitations": "The paper states performance may degrade under domain-shifted or noisy captions, and that datasets with weak semantic hierarchy may see limited gains from the hyperbolic geometry (their Appendix A5). RAHA underperforms CovMatch at the smallest budget tested (100 pairs, both Flickr30K and about-tied on COCO), so the rank-aware hyperbolic approach is not uniformly better under the most extreme compression. The per-iteration cost is substantially higher than CovMatch's at realistic batch sizes (~400s vs ~55s at batch 64) due to the SVD and hyperbolic-lifting overhead, a cost not weighed against the accuracy gain in the paper. The method inherits CovMatch's dependence on already- pretrained encoders and, like the rest of the group, is bounded by the expressivity of the teacher encoder used." }, "notes": "" }, { "id": "structural-assessment-for-understanding-2026", "type": "paper", "title": "Structural Assessment for Understanding and Guiding Dataset Distillation in Discrete Token Space", "arxiv": "2606.21705", "url": null, "date": "2026-06-19", "venue": { "name": "ECCV 2026", "status": "conference" }, "authors": [ "Yue Cao", "Jianyang Gu", "Vyacheslav Kungurtsev", "Yu Hu", "Jozsef Hamari", "Zheng Liu", "Mohsen Zardadi" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Analyzes distilled datasets through discrete visual tokenizers, introducing a structural score that measures how balanced a distilled dataset's token-level composition is, and finds that balanced token composition — not divergence from the original data distribution — correlates with higher validation performance; shows that samples with high structural scores can guide diffusion-based dataset distillation toward more effective synthetic sets.", "abstract": "Dataset distillation (DD) has proven to reduce training cost while preserving accuracy. While promising, the factors that make one distilled dataset more effective than another remain poorly understood. In this work, we investigate this question through the lens of discrete visual tokenizers. Whereas many prior DD efforts emphasize matching global data distributions, we suggest that the effectiveness depends on which semantic concepts are captured and how they are composed. Discrete visual tokenizers provide a finite vocabulary that enables direct statistical analysis of such compositional structure. Through quantitative analysis of token-level statistics, we introduce the structural score to measure the adequacy of token compositions. We observe that distilled datasets with balanced token composition yield higher validation performance. On the other hand, divergence from the original data does not necessarily harm performance. We further show that samples with high structural scores in the discrete token space can effectively guide diffusion-based DD. Our findings highlight the importance of token composition in dataset effectiveness, offering a principled complement to distributional similarity considerations in DD.", "awesome_sections": [ "Main / Better Understanding" ] }, { "id": "distill-once-adapt-life-2026", "type": "paper", "title": "Distill Once, Adapt Life-Long: Exploring Dataset Distillation for Continual Test-Time Adaptation", "short": "DO-ALL", "arxiv": "2606.20196", "url": null, "date": "2026-06-18", "venue": { "name": "ECCV 2026", "status": "conference" }, "authors": [ "Hyun-Kurl Jang", "Jihun Kim", "Hyeokjun Kweon", "Kuk-Jin Yoon" ], "kind": "application", "settings": [ "image-classification" ], "families": [], "applications": [ "test-time-adaptation" ], "tier": "notable", "links": { "code": "https://github.com/blue-531/DOALL", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Before deployment, DO-ALL distills the source data once into a small set of synthetic anchors; during continual test-time adaptation, each target sample is matched to its closest anchor, which supplies a stable reference for source replay, representation alignment and manifold-smoothing regularization without retaining raw source data. As a plug-in, it is reported to consistently improve long-term robustness of existing CTTA algorithms on CIFAR100-C, ImageNet-C and CCC.", "abstract": "Continual Test-Time Adaptation (CTTA) aims to maintain model performance under evolving target domains by adapting online without labeled data. However, practical deployments often cannot retain the source dataset due to privacy or licensing constraints, and purely source-free CTTA methods tend to become unstable under long-term distribution shift, suffering from compounding self-training errors and catastrophic forgetting. We introduce DO-ALL (Distill Once, Adapt Life-Long), a plug-and-play framework that revisits source information in a compact and privacy-conscious form via Dataset Distillation (DD). Before deployment, DO-ALL performs DD to produce a small set of synthetic distilled anchors that summarize the source distribution. During adaptation, each target sample is matched with its most semantically aligned anchor, which provides a stable reference for various CTTA via source replay, representation alignment, and manifold-smoothing regularization. DO-ALL can be seamlessly integrated into existing CTTA algorithms, consistently improving long-term robustness across CIFAR100-C, ImageNet-C, and the CCC benchmark. This demonstrates the potential of leveraging DD to enable stable and continuous adaptation without retaining raw source data. The code is available at https://github.com/blue-531/DOALL.", "awesome_sections": [ "Applications / Test-Time Adaptation" ], "notes": "families left empty: abstract does not name the underlying distillation mechanism used to build the anchors." }, { "id": "dd-vs-coresets-2026", "type": "paper", "title": "Rethinking Dataset Distillation for Classification: Do Distilled Sets Outperform Coresets?", "short": "Do distilled sets beat coresets?", "arxiv": "2606.18209", "url": null, "date": "2026-06-16", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Trisha Mittal", "Akshay Mehra", "Joshua Kimball" ], "kind": "evaluation-benchmark", "settings": [ "image-classification" ], "families": [], "tier": "notable", "links": { "code": null, "project": null }, "facets": { "labels": "mixed", "regime": "decoupled", "prior": [ "pretrained-classifier", "diffusion" ], "scale": [ "imagenet-1k", "imagenet-subsets" ] }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "label-worth-2024 (Qin et al.) and DD-Ranking showed soft labels and training tricks inflate reported DD accuracy relative to random images; RD3 showed the same for post-evaluation protocol variance among decoupled methods. But no prior work directly benchmarked current diffusion-guided and decoupled DD methods (VLCP, D3HR, Minimax, DiT-Distillation, MGD3, ManifoldGD, FADRM+) against strong coreset-selection baselines (AUM, concept-guided CCS) under one standardized protocol at ImageNet scale; most DD papers either omit CS baselines or compare only to random selection or weak coresets.", "problem": "DD papers routinely claim superiority over coreset selection on the assumption that a subset of real images is fundamentally less expressive than synthesized data, but this claim is rarely tested against strong CS methods under the same evaluation pipeline, and DD accuracy is known to be sensitive to which of several training protocols (hard-label, single-teacher, multi-teacher) is used.", "idea": "Hold the dataset, architecture, IPC and training protocol fixed and vary only whether the condensed set was built by distillation or by coreset selection; if strong coresets match or beat distilled sets under every protocol while costing far less to build and covering the data distribution better, DD's claimed intrinsic advantage over selection does not hold at ImageNet scale.", "method": "Benchmarks seven DD methods (three generative training-based: VLCP, D3HR, Minimax; three generative training-free/guided: DiT-Distillation, MGD3, ManifoldGD; one non-generative decoupled: FADRM+) against three CS methods (uniform random, AUM margin-based scoring, concept-bottleneck AUM) on ImageNet-1K, ImageNet-100 and ImageNette at IPC 10/50, training ResNet-18/ResNet-50 (plus ResNet-AP-10 on the smaller sets) from scratch under three evaluation protocols: hard label (CutMix + random-resize-crop, no teacher), single-teacher soft label (RDED's per-crop relabeling with one ResNet-18 teacher), and multi-teacher soft label (EDC's protocol: four teachers' MSE loss plus hard-label cross-entropy). CS coresets use a per-class-adapted Coverage-centric Coreset Selection (stratified sampling over an importance score after discarding the least reliable hardest samples, Alg. 1/App. B) rather than picking pure-hardest examples, since prior work shows hardest-only selection fails catastrophically at small IPC. AUM scores use one ResNet-18 trained 60 epochs on the full dataset; concept scores use a frozen CLIP/LLaVA-derived concept bottleneck trained for 100 epochs, avoiding a full-scale training run. Also measures representativeness (min-max cosine coverage of a validation set) and diversity (1 minus max intra-set cosine similarity) in a frozen ResNet-18 feature space, FID, and construction wall-clock time.", "evidence": "ImageNet-1K, ResNet-18/50, same IPC and dataset, three label regimes on the same condensed sets (Table 2) — hard label: AUM beats the best DD method by +3.3 points at IPC10 (18.9% vs. 15.6% for ManifoldGD) and by >4.2 at IPC50 (43.5% vs. 39.3%) on ResNet-18; on ResNet-50, AUM beats the best DD method (Minimax, 12.3%) by +5.0 at IPC10 (17.3%) and beats the best DD method (44.8% MGD3) by +4.1 at IPC50 (48.9%); several DD methods (VLCP, D3HR, DiT) underperform even Random coreset selection under hard labels. Single-teacher soft label (RDED protocol): gaps narrow but CS stays ahead or ties — ResNet-18 IPC50, Concepts 61.0% edges out the best DD method ManifoldGD 60.4%; ResNet-50 IPC50, AUM and Concepts both reach 66.0% vs. best DD (ManifoldGD) 65.6%. Multi-teacher soft label (EDC protocol): CS consistently ahead — ResNet-18 IPC50, AUM/Concepts 60.5-60.6% vs. best DD (ManifoldGD) 59.8%. On the small, easily-separable ImageNette (10 classes), the pattern reverses: DD methods (e.g. ManifoldGD, VLCP) beat all three CS methods under hard labels (e.g. ResNet-18 IPC10: ManifoldGD 60.0% vs. AUM 57.0% vs. Random 44.9%) — the paper's evidence that DD's advantage, where real, is confined to small/easy datasets, not ImageNet scale. Construction cost (Table 5, ImageNet-1K, excludes any teacher/generative-model pretraining): DD methods scale linearly with IPC (ManifoldGD 12.8h at IPC10 -> 51.7h at IPC50; FADRM+ 2.5h -> 12.5h) while CS is IPC-independent (AUM 6.0h at both IPC10 and IPC50; Concepts 3.0h at both). Representativeness- diversity scatter (Fig. 1b) shows AUM in the high-representativeness/high-diversity corner while DD methods match CS on representativeness but trail on diversity, and a qualitative comparison (Fig. 4, \"Bald Eagle\" class) shows AUM covering varied pose/background/lighting while VLCP/ManifoldGD repeat near-identical canonical views and FADRM+ shows visible synthesis artifacts.", "limitations": "Restricted to image classification. Restricted to ImageNet-1K and its subsets because most DD methods only release distilled sets for that dataset; several generative DD methods require training a generative model on the full dataset, which the authors note makes them impractical to extend to new domains within this evaluation. Does not include gradient/trajectory/distribution- matching or kernel-based bi-level DD methods (only decoupled and diffusion-guided/generative methods are tested) or hard-label-native coreset baselines like forgetting/k-center directly (uses AUM and concept-CCS as its two scored CS methods). AUM's own scoring pass still requires one 60-epoch ResNet-18 training run on the full dataset, an upfront cost the paper's time comparison (Table 5) does not include. The hard-label protocol applies CutMix/random-resize-crop uniformly, which the paper does not ablate against a no-augmentation control (unlike DC-BENCH)." }, "summary": "Benchmarks seven state-of-the-art dataset-distillation methods against three coreset-selection strategies on ImageNet-1K, ImageNet-100 and ImageNette under three standardized evaluation protocols, finding that some distillation methods fail to beat random subsets and that the strongest ones are comparable to or worse than coresets on these large-scale datasets while costing substantially more to construct, with coresets giving better distributional coverage.", "abstract": "Dataset distillation (DD) has emerged as a prominent approach in data centric machine learning, aiming to synthesize compact training sets for efficient training by compressing the information in large datasets into a small number of synthetic samples. However, DD methods are often evaluated under inconsistent evaluation protocols, ranging from standard ERM to single/multi-teacher supervision, making it difficult to isolate the effectiveness of distilled data from evaluation. Moreover, many prior methods claim that DD outperforms data pruning approaches such as coreset selection (CS), based on the assumption that restricting condensed datasets to subsets of real samples fundamentally limits their expressiveness. In this work, we critically evaluate DD methods through large-scale experiments using standardized datasets and evaluation protocols to assess their intrinsic effectiveness. We benchmark seven state-of-the-art (SOTA) DD methods on ImageNet-1K, ImageNet100, and ImageNette, using three widely adopted training protocols against three CS strategies. Our results show that while some DD methods fail to outperform even simple random subsets, the SOTA DD approaches are comparable to or worse than coresets on large-scale datasets and incur a substantially higher cost for construction. Beyond accuracy, we also evaluate the representativeness, diversity, and quality of condensed sets, and find that coresets consistently achieve better coverage of the original data distribution. These findings highlight the limited practical advantages of current DD methods and show that coresets remain competitive and are often a more computationally efficient alternative for data-centric learning.", "awesome_sections": [ "Main / Better Understanding" ] }, { "id": "gadc-2026", "type": "paper", "title": "Geometry-Aware Dataset Condensation for Diffusion Model Training", "short": "GADC", "arxiv": "2606.05883", "url": null, "date": "2026-06-04", "venue": { "name": "ICML 2026", "status": "conference" }, "authors": [ "Xiao Cui", "Yulei Qin", "Mo Zhu", "Wengang Zhou", "Hongsheng Li", "Houqiang Li" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "coreset-selection" ], "applications": [ "generative-model-training" ], "tier": "catalogued", "links": { "code": "https://github.com/2018cx/GADC", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Reformulates real-subset selection for training diffusion models as a geometry-aware distribution-alignment problem via one-sided partial optimal transport, allowing unmatched mass in low-density regions so the selected subset preserves the distributional geometry diffusion likelihood objectives need, complemented by feature-statistics and semantic-consistency regularizers and a two-stage discrete optimization; reports superior fidelity and distributional coverage for training diffusion models across variants, subset sizes, resolutions and training rounds.", "abstract": "Dataset condensation aims to construct compact datasets from real data via synthesis or selection. However, existing approaches are ill-suited for diffusion model training: synthetic data generation often yields low-fidelity samples unsuitable for authentic modeling, while real subset selection typically fails to preserve the distributional geometry required by diffusion likelihood objectives. To address this, we propose to reformulate real subset selection as a geometry-aware distribution alignment problem. By incorporating one-sided partial optimal transport, our method selectively aligns a compact subset with the full data distribution while allowing unmatched mass in low-density regions, ensuring the preserved geometric structure necessary for effective diffusion model training. To further ensure distributional fidelity, we complement geometric alignment with lightweight feature-statistics and semantic consistency regularization. An efficient two-stage discrete optimization strategy is proposed to achieve this alignment objective. Extensive experiments across diffusion variants, subset sizes, image resolutions, and training rounds show that our method achieves superior fidelity and distributional coverage in diffusion model training. Codes are available at https://github.com/2018cx/GADC.", "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "notes": "Reclassified from diffusion-guided-synthesis to coreset-selection: the method selects a real subset (not diffusion-sampled synthetic data) to improve diffusion-model training. The downstream learner is a generative model, not a classifier -- no setting in docs/taxonomy.md fits exactly; kept as image-classification with an applications tag. Flagged as a taxonomy gap." }, { "id": "foster-2026", "type": "paper", "title": "FOSTER: First-order Dataset Distillation for Text-based Sequential Recommendation", "short": "FOSTER", "arxiv": "2605.30772", "url": null, "date": "2026-05-29", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Hung Vinh Tran", "Tong Chen", "Xinyi Gao", "Junliang Yu", "Julien Monteil", "Hongzhi Yin" ], "kind": "method", "settings": [ "other-data" ], "families": [], "facets": { "regime": "single-level" }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Proposes FOSTER for text-based sequential-recommendation dataset distillation, replacing the bi-level gradient computation that language-model-based item encoding makes prohibitively expensive with a first-order optimization scheme using trajectory-anchored parameter resets, stochastic item-subset sampling in place of full-corpus embedding extraction, and a regularizer that encourages co-occurrence of semantically similar items in the synthetic sequences; reports it approximates full-dataset performance with as few as 20 synthetic interaction sequences across three benchmarks.", "abstract": "Text-based sequential recommender systems, while greatly improving recommendation accuracy by incorporating item contexts, are undeniably more expensive to train. By condensing a large dataset into a compact set of synthetic samples for model training, dataset distillation offers a promising solution. However, its adoption in text-based sequential recommendation is non-trivial given the large pool of discrete items. This challenge is further compounded by language model-based item encoding, which makes bi-level optimization commonly used in dataset distillation prohibitively expensive. To this end, we propose First-order dataset distillation for Text-based Sequential Recommendation (FOSTER), which facilitates effectiveness and efficiency via three novel components: (1) stochastic item subset sampling that replaces costly full-corpus embedding extraction at each distillation step; (2) first-order optimization with trajectory-anchored parameter reset to avoid expensive bi-level gradient computation; and (3) regularization that explicitly promotes co-occurrence between semantically similar items in the synthetic sequences. Extensive experiments on three benchmarks show that FOSTER consistently outperforms existing dataset distillation and coreset selection baselines, approximating full-dataset performance using as few as 20 synthetic interaction sequences.", "awesome_sections": [ "Applications / Recommender Systems" ], "notes": "\"Trajectory-anchored parameter reset\" is not a clean match to any P1-P4 family as described in the abstract; left empty and flagged as a doubt." }, { "id": "d3s2-2026", "type": "paper", "title": "D3S2: Diffusion-Guided Dataset Distillation for Semantic Segmentation", "short": "D3S2", "arxiv": "2605.25022", "url": null, "date": "2026-05-24", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Wenjie Zheng", "Haoji Hu", "Jiali Lu", "Xingze Zou", "Jing Wang" ], "kind": "method", "settings": [ "dense-prediction" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "prior": [ "diffusion" ], "regime": "training-free" }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Proposes D3S2 for dense-prediction dataset distillation in semantic segmentation: a class-balanced set of masks is selected by a greedy strategy that prioritizes underrepresented classes, then a pretrained layout-to-image diffusion model generates images conditioned on those masks, guided at sampling time by a segmentation-consistency loss and a class-wise feature-matching loss for pixel-level alignment; at a 1% compression rate with Mask2Former (Swin-S) it reports 24.99% mIoU on ADE20K and 35.49% on COCO-Stuff, improving over random selection by 9.34 and 5.70 points respectively.", "abstract": "Dataset distillation (DD) aims to compress large-scale datasets into compact synthetic sets while preserving training efficacy. However, existing studies mainly focus on image classification, leaving dense prediction tasks such as semantic segmentation largely underexplored. In this work, we identify three key challenges for segmentation DD: (i) long-tailed class imbalance, (ii) the need for strict pixel-wise alignment between images and dense labels, and (iii) the high computational cost of optimizing high-resolution data with complex models. To address these challenges, we propose D3S2, a Diffusion-guided Dataset Distillation framework for Semantic Segmentation. Our method adopts a two-stage design. In Class-Balanced Mask Selection, we construct a representative mask set via a greedy strategy that prioritizes underrepresented classes. In Diffusion-Guided Image Synthesis, we employ a pretrained layout-to-image diffusion model to generate images conditioned on the selected masks, naturally ensuring spatial alignment. To further enhance the training utility of synthesized data, we introduce guided diffusion sampling with two complementary objectives: a segmentation-consistency loss for pixel-level alignment, and a class-wise feature matching loss for aligning per-class feature statistics across layers. Extensive experiments demonstrate the superiority of D3S2. Notably, at an extremely compression rate of 1%, our method achieves 24.99% and 35.49% mIoU on ADE20K and COCO-Stuff with Mask2Former (Swin-S), outperforming random selection by 9.34% and 5.70%, respectively.", "awesome_sections": [ "Applications / Semantic Segmentation" ], "notes": "" }, { "id": "mdm-2026", "type": "paper", "title": "Multimodal Distribution Matching for Vision-Language Dataset Distillation", "short": "MDM", "arxiv": "2605.23482", "url": null, "date": "2026-05-22", "venue": { "name": "CVPR 2026", "status": "conference" }, "authors": [ "Jongoh Jeong", "Hoyong Kwon", "Minseok Kim", "Kuk-Jin Yoon" ], "kind": "method", "settings": [ "vision-language" ], "families": [ "distribution-matching" ], "facets": { "labels": "hard", "prior": [ "pretrained-classifier", "trained-experts" ], "regime": "single-level", "parameterization": [ "pixels", "embedding-features" ], "scale": [ "flickr30k-coco" ] }, "tier": "notable", "links": { "code": "https://github.com/andyj1/mdm", "project": null }, "status": { "verified": true, "verified_on": "2026-09-14", "source": "arxiv-api", "depth": "full-text" }, "summary": "MDM distills vision-language pairs by initializing synthetic image-text pairs from clusters in the joint embedding space, forming a mixed teacher by interpolating independently fine-tuned models in weight space, and matching real and synthetic joint distributions on the unit hypersphere with a geometry-aware objective that exploits cross-modal agreement and discrepancy directions alongside symmetric contrastive learning. The paper reports compact synthetic sets that preserve multimodal semantics and remain robust under cross-architecture evaluation on image-text retrieval benchmarks.", "abstract": "Dataset distillation compresses large training sets into compact synthetic datasets while preserving downstream performance. As modern systems increasingly operate on paired vision-language inputs, multimodal distillation must preserve representation quality and cross-modal alignment under tight compute and memory budgets, yet prior methods often require heavy computes and overlook their correlations. To address this, we present Multimodal Distribution Matching (MDM), a geometry-aware framework for efficient and generalizable multimodal distillation. Specifically, MDM integrates complementary components at the data, model, and loss levels. At the data level, it initializes synthetic image-text pairs by sampling from clusters in the joint embedding space. At the model level, it forms a mixed teacher by interpolating independently fine-tuned models in weight space according to their angular deviation from the pretrained anchor. At the loss level, it matches joint distributions on the unit hypersphere using a geometry-aware matching objective that exploits the joint features in the cross-modal agreement and discrepancy directions along with symmetric contrastive learning. Across image-text retrieval benchmarks with cross-architecture evaluation, MDM yields compact synthetic sets that preserve multimodal semantics, substantially reduce distillation cost, and remain robust across architectures.", "builds_on": [ "mtt-vl-2024", "lors-2024" ], "explained": { "before": "MTT-VL and LoRS both distill vision-language pairs via bi-level trajectory matching, repeatedly generating and matching teacher/student parameter trajectories from real and synthetic data.", "problem": "The paper diagnoses two costs of trajectory matching: it requires \"substantial computational and memory overhead\" from repeated bi-level optimization with nested gradients over stored expert trajectories, and it biases the synthetic data toward the specific architecture whose training dynamics are being matched, which the paper connects to trajectory-matching methods' weak cross-architecture generalization; the paper also argues prior methods do not explicitly preserve the joint image-text distributional structure (both modality-shared and modality-specific information) under tight pair budgets.", "idea": "Replace trajectory replay with direct distribution matching in a joint embedding space: initialize synthetic pairs by clustering the joint (concatenated image+text) embedding space of the real data rather than randomly, build a single \"mixed teacher\" by interpolating independently fine-tuned experts in weight space (weighted by how much they agree, so disagreement falls back toward the pretrained anchor), and match real versus synthetic distributions using a geometry-aware kernel that operates on the unit hypersphere along both a cross-modal agreement direction and a discrepancy direction, instead of matching means as in plain distribution matching (DM).", "method": "Synthetic image pixels ($3\\times224\\times224$) and 768-d text embeddings are optimized. A frozen encoder $\\Psi$ (NFNet image + BERT text + projection heads) provides features but is re-initialized per iteration by weight-space interpolation of $N{=}2$ fine-tuned experts and a pretrained anchor $\\theta_0$ (Eq. 5): per layer, the merge coefficient $t_\\ell$ is set by the cosine-similarity-derived angular agreement between the two experts' displacements from the anchor, so more agreement gives more weight to the merged expert direction and more disagreement falls back toward the shared pretrained anchor. Synthetic pairs are initialized (Eq. 4) by k-means clustering the real data's joint embeddings $[z_v;z_t]$ into $K=|\\mathcal D_{syn}|$ clusters and picking, per cluster, the real pair closest to the centroid. The matching objective (Eq. 6-11) forms an agreement direction $u=\\text{normalize}(z_v+z_t)$ and a discrepancy direction $g=\\text{normalize}(z_v-z_t)$ per pair, then computes a Geodesic Kernel Energy (an MMD-style two-sample statistic using an angular/geodesic Gaussian kernel $k_{geo}(a,b)=\\exp(-\\phi(a,b)^2/2 \\sigma^2)$ with $\\phi$ the arccos angular distance) between the real and synthetic sets' agreement distributions ($\\mathcal L_{agr}$) and discrepancy distributions ($\\mathcal L_{dis}$) separately; the total loss $\\mathcal L_{MDM}=\\mathcal L_{InfoNCE}+\\lambda_{agr}\\mathcal L_{agr}+\\lambda_{dis}\\mathcal L_{dis}$ adds a standard bidirectional InfoNCE term to keep synthetic pairs themselves cross-modally aligned. Both encoders are used only as fixed (per-iteration) feature extractors during distillation - there is no expert-trajectory replay. At evaluation, a freshly initialized model is trained on the synthetic pairs for 100 epochs.", "evidence": "NFNet+BERT protocol, Table 1. Flickr30K@100: MDM (Ours) IR@1/5/10 8.1/24.7/36.2, TR@1/5/10 11.5/32.6/45.0, vs LoRS 8.3/24.1/35.1, 11.8/35.8/49.2 (roughly matched on IR@1/TR@1, ahead on IR@5/10, behind on TR@5/10) and vs MTT-VL 4.7/15.7/24.6, 9.9/28.3/39.1. @200: MDM IR 9.1/26.7/39.1, TR 13.0/33.7/47.4 vs LoRS IR 8.6/25.3/36.6, TR 14.5/38.7/53.4. @500: MDM IR 10.0/29.3/42.0, TR 13.7/37.0/51.5 vs LoRS IR 10.0/28.9/41.6, TR 15.5/29.8/53.7 (near-tied on IR, LoRS ahead on TR@1/10). COCO@100: MDM IR 1.9/7.6/13.2, TR 3.6/13.7/21.6 vs LoRS IR 1.8/7.1/12.2, TR 3.3/12.2/19.6. @500: MDM IR 3.7/13.6/22.2, TR 5.6/18.4/28.2 vs LoRS IR 2.8/9.9/16.5, TR 5.3/18.3/27.9. Cross-architecture generalization (Table 2, distilled on NFNet+BERT, evaluated on other encoders, Flickr30K@100 mean recall): MDM 16.1 vs LoRS 9.9; COCO@100: MDM 6.1 vs LoRS 2.5. Cost (Table 3, Flickr8K): distillation time falls from LoRS's 76.9/137.5/206.4 minutes (100/200/500 pairs) to MDM's 5.7/7.97/3.68 minutes (93-98% less), and MDM converges in roughly 50-200 iterations versus LoRS's 850-2350 (Fig. 4). The key ablations (Table 4a-b, Table 5, Flickr8K@100) isolate three mechanisms: joint-space k-means initialization beats random-sample, image-only, or text-only clustering by about 1.1-1.3 mean-recall points (21.9 vs 20.6-20.8); angle-guided weight-space expert merging beats a naive random weighted sum by 2.3 points (21.9 vs 19.6) and beats using only a single fine-tuned expert (17.8) or only the pretrained anchor (6.8) by much larger margins; and adding the discrepancy loss $\\mathcal L_{dis}$ contributes more than the agreement loss $\\mathcal L_{agr}$ alone (21.53 vs 21.02 over the InfoNCE-only baseline's 20.98), with both together giving the best result (21.94).", "limitations": "The paper states MDM still assumes access to pretrained image and text encoders (and, for the mixed-teacher mechanism, multiple independently fine-tuned experts), so it does not remove that dependency, only the trajectory-storage cost built on top of it. At the 200-pair Flickr30K budget MDM's mean text-retrieval numbers trail LoRS's (e.g. TR@10 47.4 vs 53.4), a result the paper acknowledges without fully explaining. Distilled text remains 768-d embeddings rather than readable captions. Distilled images show the same noise-like high-frequency artifacts reported in prior distillation work. The method's teacher-interpolation mechanism depends on the quality and diversity of the fine-tuned experts used, a dependency the paper does not quantify." }, "notes": "" }, { "id": "mind-your-margin-and-2026", "type": "paper", "title": "Mind Your Margin and Boundary: Are Your Distilled Datasets Truly Robust?", "short": "C^2R", "arxiv": "2605.20606", "url": null, "date": "2026-05-20", "venue": { "name": "ICML 2026", "status": "conference" }, "authors": [ "Muquan Li", "Yingyi Ma", "Yihong Huang", "Hang Gou", "Ke Qin", "Ming Li", "Yuan-Fang Li", "Tao He" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "applications": [ "adversarial-robustness" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Argues prior robust dataset distillation trades off accuracy for robustness poorly because it treats all adversarial examples uniformly and does not explicitly widen inter-class decision-boundary separation; C^2R instead derives a robust-margin perturbation score to build an attack-aware curriculum that prioritizes the smallest-margin adversaries driving robust error, paired with a class-balanced contrastive robustness loss that widens boundary separation. Reports the best robust accuracy across six attacks on CIFAR-10/100, Tiny-ImageNet and ImageNet-1K subsets, beating prior robust DD by 2.8% on average.", "abstract": "Dataset distillation (DD) compresses a large training set into a small synthetic set for efficient training, but most DD methods optimize only clean accuracy and leave robustness uncontrolled. Recent robust DD methods improve robustness, yet they often suffer from a poor accuracy-robustness trade-off because they (i) treat all adversarially perturbed examples uniformly, despite robust risk being dominated by near-zero robust margins, and (ii) do not explicitly increase inter-class separation in the decision boundary where attacks concentrate. We present Contrastive Curriculum for Robust Dataset Distillation (C^2R), a framework that couples an attack-aware curriculum with a contrastive robustness objective. From a robust-margin perspective, we derive a perturbation score that approximates each sample's robust hinge, enabling a curriculum that prioritizes the smallest-margin adversaries that most directly drive robust error. In parallel, a class-balanced contrastive robustness loss enforces adversarial invariance while explicitly widening boundary separation across classes. Experiments on CIFAR-10/100, Tiny-ImageNet, and multiple ImageNet-1K subsets under six attacks show that C^2R achieves the best robust accuracy, outperforming prior robust DD by 2.8% on average.", "awesome_sections": [ "Applications / Robustness" ], "notes": "" }, { "id": "graph-condensation-reset-2026", "type": "paper", "title": "Position: Graph Condensation Needs a Reset -- Move Beyond Full-dataset Training and Model-Dependence", "short": "Graph condensation needs a reset", "arxiv": "2605.18893", "date": "2026-05-17", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Mridul Gupta", "Samyak Jain", "Vansh Ramani", "Hariprasad Kodamana", "Sayan Ranu" ], "kind": "survey", "settings": [ "graph" ], "families": [], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-14", "source": "arxiv-api", "depth": "abstract" }, "summary": "A position paper arguing that mainstream graph condensation contradicts its own goal: gradient-matching methods must train on the full graph to produce the condensed one, generalize poorly across GNN architectures, and are judged by node compression ratios that ignore real resource savings and condensation overhead. It calls for lightweight, architecture-agnostic methods and resource-based evaluation.", "abstract": "Graph Neural Networks (GNNs) are powerful tools for learning from graph-structured data, but their scalability is increasingly strained by the size of real-world graphs in domains like recommender systems, fraud detection, and molecular biology. Graph condensation -- the task of generating a smaller synthetic graph that retains the performance of models trained on the original -- has emerged as a promising solution. However, the dominant approach of gradient matching introduces a fundamental contradiction: it requires training on the full dataset to create the compressed version, thereby undermining the goal of efficiency. Worse still, these methods suffer from high computational overhead, poor generalization across GNN architectures, and brittle reliance on specific model configurations. Equally concerning is the community's reliance on misleading evaluation protocols such as node compression ratios, which fail to reflect true resource savings, condensation overhead, and illusory application to neural architecture search. These shortcomings are not incidental -- they are systemic, and they obstruct meaningful progress. In this position paper, we argue that graph condensation, in its current form, needs a reset. We call for moving beyond full-dataset training and model-dependent design, and instead advocate for methods that are lightweight, architecture-agnostic, and practically deployable. By identifying key methodological flaws and outlining concrete research directions, we aim to reorient the field toward approaches that deliver on the true promise of condensation: efficient, generalizable, and usable GNN training at scale." }, { "id": "from-compression-to-accountability-2026", "type": "paper", "title": "From Compression to Accountability: Harmless Copyright Protection for Dataset Distillation", "short": "SubPopMark", "arxiv": "2605.12942", "url": null, "date": "2026-05-13", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Yan Liang", "Ziyuan Yang", "Mengyu Sun", "Joey Tianyi Zhou", "Yi Zhang" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [], "applications": [ "copyright-protection" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Observes that networks trained on distilled data still memorize subpopulation-level prediction biases, and uses this to build SubPopMark, a harmless (non-backdoor) copyright and provenance marker for distilled datasets: a Copyright Verification Marker stage injects a class-consistent subpopulation bias while preserving the original distillation trajectory, and a User-Specific Tracing Marker stage adds user-distinguishable perturbations on top. Provenance is verified black-box by comparing a suspicious model's output-behavior signature against a reference bank built from standard and subpopulation-shifted test sets.", "abstract": "Large-scale datasets have been a key driving force behind the rapid progress of deep learning, but their storage, computational, and energy costs have become increasingly prohibitive. Dataset distillation (DD) mitigates this problem by synthesizing compact yet informative datasets, thereby enabling efficient model training and storage. However, the ease of copying and distributing distilled datasets introduces serious risks of copyright infringement and data leakage. Existing protection methods are primarily designed for raw datasets rather than distilled datasets, and typically rely on backdoor-triggered malicious behaviors, which may raise security concerns. In this paper, we observe that deep neural networks tend to memorize subpopulation distributions during training, resulting in a systematic prediction bias, where models perform better on samples aligned with memorized subpopulations. Motivated by this observation, we propose SubPopMark, a harmless subpopulation-driven protection framework for distilled datasets. SubPopMark consists of two stages. First, the Copyright Verification Marker(CVM) optimization stage injects a class-consistent subpopulation bias while preserving the original optimization trajectory. Second, the User-Specific Tracing Marker (USTM) optimization stage further introduces user-distinguishable perturbations into the CVM-augmented data. To enable black-box verification and tracing, we construct a reference behavior bank by collecting model outputs over carefully designed test sets that cover both standard and subpopulation-shifted data distributions. The provenance of a suspicious model is then inferred by comparing its output behavior signature with the bank and identifying the most consistent reference behavior pattern.", "notes": "Marking layer added atop an unspecified underlying condensation method; families left empty." }, { "id": "diver-2026", "type": "paper", "title": "DIVER:Diving Deeper into Distilled Data via Expressive Semantic Recovery", "short": "DIVER", "arxiv": "2605.12649", "url": null, "date": "2026-05-12", "venue": { "name": "ICML 2026", "status": "conference" }, "authors": [ "Qianxin Xia", "Zhiyong Shu", "Wenbo Jiang", "Jiawei Du", "Jielei Wang", "Guoming Lu" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "prior": [ "diffusion" ], "regime": "training-free", "parameterization": [ "pixels" ], "scale": [ "imagenet-subsets", "imagenet-1k" ] }, "tier": "catalogued", "links": { "code": "https://github.com/einsteinxia/DIVER", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Takes distilled images from any prior single-stage method and 'dives deeper' with a pretrained diffusion model in three steps -- semantic inheritance projects the abstract distilled images' high-level semantics into the diffusion latent space to filter out architecture-specific noise, semantic guidance directs the reverse process to preserve that semantics, and semantic fusion applies the guidance only during the concrete phase to avoid artifacts -- improving cross-architecture generalization with runtime comparable to a raw DiT pass on ImageNet at 4GB GPU memory.", "abstract": "Dataset distillation aims to synthesize a compact proxy dataset that is unreadable or non-raw from the original dataset for privacy protection and highly efficient learning. However, previous approaches typically adopt a single-stage distillation paradigm, which suffers from learning specific patterns that overfit on a prior architecture, consequently suppressing the expression of semantics and leading to performance degradation across heterogeneous architectures. To address this issue, we propose a novel dual-stage distillation framework called ${\\textbf{DIVER}}$, which leverages the pre-trained diffusion model to dive deeper into $\\textbf{DI}$stilled data $\\textbf{V}$ia $\\textbf{E}$xpressive semantic $\\textbf{R}$ecovery, an entire process of semantic inheritance, guidance, and fusion. Semantic inheritance distills high-level semantics of abstract distilled images into the latent space to filter out architecture-specific ``noise\" and retain the intrinsic semantics. Furthermore, semantic guidance improves the preservation of the original semantics by directing the reverse procedure. Finally, semantic fusion is designed to provide semantic guidance only during the concrete phase of the reverse process, preventing semantic ambiguity and artifacts while maintaining the guidance information. Extensive experiments validate the effectiveness and efficiency of DIVER in improving classical distillation techniques and significantly improving cross-architecture generalization, requiring processing time comparable to raw DiT on ImageNet (256$\\times$256) with only 4 GB of GPU memory usage.", "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "builds_on": [ "mtt-2022", "dm-2023", "sre2l-2023", "g-vbsm-2024", "d4m-2024", "mgd3-2025", "glad-2023", "ncfm-2025" ], "explained": { "before": "Single-stage distillation methods -- meta/gradient/trajectory matching (MTT), distribution matching (DM, NCFM), latent-optimization (GLaD), decoupled inversion (SRe2L, G-VBSM), and diffusion-guided synthesis (D4M, MGD3) -- each fix the distilled images once, optimized against or generated for one particular architecture's dynamics or statistics.", "problem": "Because a single-stage distilled image is shaped by one architecture's gradients, BN statistics, or guidance signal, it overfits architecture-specific patterns and suppresses the transferable, high-level semantics that would let a model trained on it generalize to unseen architectures -- observed as consistent cross-architecture accuracy drops for prior methods' distilled sets.", "idea": "Treat any existing distilled image as a rough semantic sketch and use a second, purely post-hoc stage with a frozen pretrained diffusion model to \"dive deeper\": project the distilled image into the diffusion latent space just far enough to strip architecture-specific noise while keeping its semantics recoverable, then run guided denoising that pulls the trajectory back toward the original image's semantics, restricting that guidance to the middle (\"semantic\") phase of the reverse process so it neither collapses to the input nor drifts into label-irrelevant artifacts.", "method": "A frozen DiT-XL/2 (256x256) with a frozen VAE (vae-ft-mse) is the only diffusion model; nothing is trained. Semantic Inheritance encodes the distilled image $z_0$ into the VAE latent space and adds forward-process noise for $t_f=25$ steps ($z_{t_f}=\\sqrt{\\alpha_{t_f}}z_0+\\sqrt{1-\\alpha_{t_f}}\\epsilon$), chosen to balance approaching a Gaussian prior against retaining recoverable features. Semantic Guidance adds a term $G_t=(\\hat z_t-z_0)^2\\cdot\\sigma_t/2$ to the reverse process that penalizes drift of the denoised estimate away from the original distilled latent. Semantic Fusion restricts this class- label-conditioned guidance to a semantic phase window $t\\in[t_l,t_h]=[25,40]$ of the 50-step DDIM schedule (classifier-free guidance $\\gamma=0.1$), rather than the full trajectory, to avoid the artifacts a full-trajectory guidance signal would introduce. The method takes any prior method's already-distilled images as input and requires no access to the original real dataset.", "evidence": "ImageNet-subsets 128x128 cross-architecture (Table 1), MTT baseline vs +DIVER, IPC10 (e.g. ImageSquawk): 21.7+/-2.5 to 33.8+/-1.4; ImageYellow IPC10: 19.1+/-0.9 to 34.8+/-1.6. Full ImageNet-1K 224x224 (Table 3): SRe2L IPC50, RN50: 55.6+/-0.3 to 61.1+/-0.2; G-VBSM IPC50, RN50: 58.7+/-0.3 to 64.2+/-0.4. Diffusion- based methods (Table 4), ResNet-18: MGD3 IPC10 45.8+/-0.3 to MGD3+DIVER 46.4+/-0.3; MGD3+Minimax IPC50 58.6+/-0.3 to MGD3+Minimax+DIVER 61.0+/-0.2. Table 5, ImageNette IPC50: D4M 70.2 to D4M+DIVER 75.1; MGD3 79.5 to MGD3+DIVER 81.2. Baseline numbers for MTT/SRe2L/G-VBSM are taken directly from those methods' public distilled datasets (copied); D4M/MGD3/Minimax numbers are produced by running each method's official code (re-run). No D3HR, CaO2 or IGD comparison. Cost: 2.48s/image on one RTX-4090 at 4GB memory, architecture-agnostic since the guidance stage never touches an evaluation network. Ablation (Table 6, ImageFruit, MTT, IPC1): SI alone 19.5+/-1.4, SG alone 20.4+/-1.7, SI+SG 21.1+/-1.9, SI+SG+SF 22.3+/-1.8, each component adding incrementally; the forward-step count $t_f=25$ and guidance factor $\\gamma=0.1$ are each shown optimal via sweep (Fig. 4).", "limitations": "Stated: \"heavily dependent on the quality of the distilled images\" it is given as input, and \"extended to a limited range of diffusion-based approaches\" (tested on D4M/MGD3, not the training-free or vision-language-conditioned branches). Observed: it is not a standalone generative method but a post-processing stage bolted onto seven different prior pipelines, so its numbers are always framed as deltas and never compared head-to-head against CoDA/DAP/ManifoldGD/VLCP/EVLF as an independent method; applying it to diffusion-based (D4M/MGD3) distilled sets shows smaller and sometimes negative gains for ConvNet-family evaluation (an \"acceptable trade-off\" the paper accepts for the cross-architecture gain); still depends on a large pretrained DiT-XL/VAE at inference time for every image." } }, { "id": "closed-form-linear-probe-2026", "type": "paper", "title": "Closed-Form Linear-Probe Dataset Distillation for Pre-trained Vision Models", "short": "CLP-DD", "arxiv": "2605.07194", "url": null, "date": "2026-05-08", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Bincheng Peng", "Guang Li", "Ping Liu", "Takahiro Ogawa", "Miki Haseyama" ], "kind": "method", "settings": [ "self-supervised" ], "families": [ "kernel-closed-form" ], "facets": { "regime": "bilevel", "prior": [ "pretrained-classifier" ], "scale": [ "imagenet-subsets", "imagenet-1k" ] }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Proposes CLP-DD for distilling data used to train linear probes on frozen pretrained vision encoders: a sample-space kernel-ridge solver gives the linear probe induced by the synthetic set in closed form (exploiting that frozen-feature linear probing has an exact solution, with no NTK approximation or inner-loop trajectory), and the synthetic images are then updated by a temperature-scaled cross-entropy evaluating that induced classifier on real features. Reports that pairing the closed-form solver with a discriminative outer loss (rather than MSE) is decisive, and that on ImageNet-1K, CLP-DD matches or surpasses trajectory-based LGM-with-DSA on three of four backbones while running about 14x faster and using under one-eighth the GPU memory.", "abstract": "Dataset distillation compresses a large training set into a small synthetic set that preserves downstream training utility. While most existing methods target training networks from scratch, modern visual transfer learning often uses frozen pre-trained encoders followed by lightweight linear probing. Existing distillation methods for this setting either unroll iterative linear-probe updates with trajectory-based gradient matching, or rely on closed-form formulations originally designed for from-scratch training with neural-tangent-kernel (NTK) approximations. Neither route exploits the fact that frozen-feature linear probing admits a closed-form solution determined directly by the pre-trained features themselves, with no infinite-width approximation and no inner-loop trajectory. We propose Closed-Form Linear-Probe Dataset Distillation (CLP-DD), a bilevel formulation that computes the linear probe induced by the synthetic set with a sample-space kernel ridge solver. The synthetic images are then updated by evaluating this induced classifier on real features through a temperature-scaled softmax cross-entropy, where the classifier columns act as learned class anchors in feature space. We further show that the choice of outer objective is decisive: pairing the closed-form inner solver with a standard MSE outer loss substantially underperforms trajectory-based methods, while the discriminative outer loss closes most of the gap. On ImageNet-100 with four pre-trained backbones, CLP-DD substantially improves over LGM without DSA and approaches LGM with DSA at a fraction of the computational cost. On ImageNet-1K, CLP-DD matches or surpasses LGM with DSA on three of four backbones while running roughly $14\\times$ faster and using less than one-eighth of the GPU memory.", "awesome_sections": [ "Main / Self-Supervised Distillation" ], "notes": "" }, { "id": "dmgd-2026", "type": "paper", "title": "DMGD: Train-Free Dataset Distillation with Semantic-Distribution Matching in Diffusion Models", "short": "DMGD", "arxiv": "2605.03877", "url": null, "date": "2026-05-05", "venue": { "name": "CVPR 2026", "status": "conference" }, "authors": [ "Qichao Wang", "Yunhong Lu", "Hengyuan Cao", "Junyi Zhang", "Min Zhang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "prior": [ "diffusion" ], "regime": "training-free", "parameterization": [ "pixels" ], "scale": [ "imagenet-subsets", "imagenet-1k" ] }, "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Proposes training-free Dual Matching Guided Diffusion: Semantic Matching via conditional-likelihood optimization removes the need for an auxiliary classifier, a dynamic guidance mechanism balances diversity against semantic alignment, and an optimal-transport-based Distribution Matching term aligns generated samples with the target distribution's structure, with two efficiency strategies (Distribution Approximate Matching, Greedy Progressive Matching); reports average accuracy gains of 2.1%, 5.4% and 2.4% on ImageNet-Woof, -Nette and -1K over fine-tuning-based diffusion methods.", "abstract": "Dataset distillation enables efficient training by distilling the information of large-scale datasets into significantly smaller synthetic datasets. Diffusion based paradigms have emerged in recent years, offering novel perspectives for dataset distillation. However, they typically necessitate additional fine-tuning stages, and effective guidance mechanisms remain underexplored. To address these limitations, we rethink diffusion based dataset distillation and propose a Dual Matching Guided Diffusion (DMGD) framework, centered on efficient training-free guidance. We first establish Semantic Matching via conditional likelihood optimization, eliminating the need for auxiliary classifiers. Furthermore, we propose a dynamic guidance mechanism that enhances the diversity of synthetic data while maintaining semantic alignment. Simultaneously, we introduce an optimal transport (OT) based Distribution Matching approach to further align with the target distribution structure. To ensure efficiency, we develop two enhanced strategies for diffusion based framework: Distribution Approximate Matching and Greedy Progressive Matching. These strategies enable effective distribution matching guidance with minimal computational overhead. Experimental results on ImageNet-Woof, ImageNet-Nette, and ImageNet-1K demonstrate that our training-free approach achieves significant improvements, outperforming state-of-the-art (SOTA) methods requiring additional fine-tuning by average accuracy gains of 2.1%, 5.4%, and 2.4%, respectively.", "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "builds_on": [ "d4m-2024", "mgd3-2025", "minimax-2024" ], "explained": { "before": "Minimax fine-tunes a diffusion model per dataset for representativeness/diversity; D4M and MGD3 guide a frozen generator with clustering-derived prototypes or modes but treat each generated sample independently and (per this paper's critique) can overemphasize invalid modes; IGD adds a classifier trained on gradient trajectories for guidance, at the cost of extra training.", "problem": "Diffusion-guided distillation methods either need an added fine-tuning stage (Minimax) or use clustering-based conditioning (D4M, MGD3) that neglects the interrelationships among the samples being generated for the same class, producing diversity deficiencies and, per the paper's mean-matching analysis, samples that over-concentrate near a class mean rather than covering the target distribution.", "idea": "Guide an unmodified pretrained diffusion model (DiT-XL) with two decoupled, training-free signals computed at sampling time: a \"Semantic Matching\" term that staggers classifier-free-style soft-label guidance across the denoising trajectory (stochastic exploration, then dynamic soft-label guidance, then semantic refinement), and a \"Distribution Matching\" term that pulls the evolving synthetic set toward the target distribution's structure via an optimal-transport objective, made tractable with two approximations (K-means clustering of the target, and greedy progressive matching that freezes already generated samples so later ones are pushed to cover what is missing rather than collapsing to the mean).", "method": "Sampling proceeds in three guidance stages by timestep window (t>=45 stochastic exploration, t in [25,45] dynamic soft-label guidance with Gaussian-noised and cross-class-perturbed labels for diversity, t<=25 semantic refinement). A second guidance term adds the gradient of an optimal-transport distance $\\nabla_{z_t}\\mathcal{L}_{OT}(P_S^t,P_T)$ between the current synthetic latent distribution $P_S^t$ and the target $P_T$; computing this exactly over the full target set is replaced by (a) Distribution Approximate Matching, clustering the target into K=10 clusters per class to approximate $P_T$ cheaply, and (b) Greedy Progressive Matching, generating samples sequentially and freezing earlier ones so each new sample is guided toward the still-uncovered part of the distribution. Nothing is fine-tuned; the DiT backbone, VAE and any teacher/classifier are frozen. Only pixels (plus labels per the evaluation protocol) are stored as the distilled artifact.", "evidence": "Hard-label protocol, ResNet10-AP (Table 1): ImageNet-Woof IPC10/50: 40.8+/-1.1 / 60.1+/-0.8 vs MGD3 40.4+/-1.9 / 56.5+/-0.8; ImageNet-Nette IPC10/50: 68.4+/-0.2 / 80.6+/-0.5 vs MGD3 66.4+/-2.4 / 79.5+/-1.3. Plugged onto Minimax's fine-tuned backbone (Minimax+Ours), Woof reaches 42.4+/-0.5 (IPC10) / 60.8+/-0.2 (IPC50). ImageNet-1K soft-label protocol, ResNet-18 (Table 2): IPC10/50: 46.3+/-0.8 / 61.4+/-0.6, vs RDED 42.0+/-0.2 (IPC10) and MGD3 60.2+/-0.1 (IPC50), Minimax 44.3+/-0.5 (IPC10); ResNet-101 IPC10/50: 50.6+/-1.2 / 68.4+/-0.4 vs RDED 48.3+/-1.0 (IPC10) / MGD3 67.7+/-0.4 (IPC50). D3HR, CaO2 and IGD are not included in the main numerical comparison tables (IGD is discussed only in related work, criticized for needing extra classifier-trajectory training); baselines are re-run under the paper's own hard-label/soft-label protocols on a shared DiT/ResNet10-AP or ResNet-18/101 setup rather than quoted verbatim from each source paper. Cost: Minimax's fine-tuning takes ~0.7 GPU-hours versus DMGD's Distribution Approximate Matching at 0.03s per class; a full ImageNet-Woof IPC50 set generates in 0.26 hours; per-image sampling cost is 1.65s vs the unguided DiT baseline's 1.49s. Ablation (Table 3, Woof): Semantic Matching alone 38.9+/-1.2 (IPC10) / 59.3+/-0.4 (IPC50); Distribution Matching alone 41.6+/-1.1 / 56.8+/-0.2; combined 40.8+/-1.1 / 60.1+/-0.8 -- showing SM dominates at low IPC (where per-sample semantic fidelity matters most) and DM dominates at high IPC (where distribution coverage matters most), and combining them is not simply additive.", "limitations": "Stated: the paper's own limitations discussion is brief and mostly frames the per-image sampling overhead (1.65s vs 1.49s baseline) as the main cost of adding two guidance terms. Observed: no comparison to D3HR, CaO2 or IGD with numbers under a shared protocol, so its ranking against the inversion/latent-statistics and influence-guided branches of the family is untested; the OT-based Distribution Matching term still requires access to the full target dataset's latents at synthesis time (via K-means over real data), so it does not remove the family's general dependence on real-data statistics even though it removes fine-tuning; K=10 clusters and the timestep boundaries (25, 45) are fixed hyperparameters not shown to transfer across datasets without retuning." } }, { "id": "fair-dataset-distillation-via-2026", "type": "paper", "title": "Fair Dataset Distillation via Cross-Group Barycenter Alignment", "short": "COBRA", "arxiv": "2605.00185", "url": null, "date": "2026-04-30", "venue": { "name": "ICML 2026", "status": "conference" }, "authors": [ "Mohammad Hossein Moslemi", "Nima Hosseini Dashtbayaz", "Zhimin Mei", "Bissan Ghaddar", "Boyu Wang" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "applications": [ "fairness" ], "tier": "notable", "links": { "code": "https://github.com/mhmoslemi/COBRA", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Argues that fairness gaps in distilled data come from mismatches between demographic subgroups' predictive patterns, not just group-size imbalance, and so persist even after correcting for imbalance; proposes distilling toward a group-imbalance-agnostic barycenter of the predictive information so subgroups induce similar representations. The plug-in is compatible with existing distillation methods and reported to substantially reduce the bias dataset distillation introduces.", "abstract": "Dataset Distillation aims to compress a large dataset into a small synthetic one while maintaining predictive performance. We show that as different demographic groups exhibit distinct predictive patterns, the distillation process struggles to simultaneously preserve informative signals for all subgroups, regardless of whether group sizes are mildly or severely imbalanced. Consequently, models trained on distilled data can experience substantial performance drops for certain subgroups, leading to fairness gaps. Crucially, these gaps do not disappear by merely correcting group imbalance, since they stem from fundamental mismatches in subgroup predictive patterns rather than from sample-size disparities alone. We therefore formally analyze the interaction between these two sources of bias and cast the solution as identifying a group-imbalance-agnostic barycenter of the predictive information that induces similar representations across all subgroups. By distilling toward this shared aggregate representation, we show that group fairness concerns can be reduced. Our approach is compatible with existing distillation methods, and empirical results show that it substantially reduces bias introduced by dataset distillation. Code is available at https://github.com/mhmoslemi/COBRA.", "awesome_sections": [ "Applications / Fairness" ], "notes": "" }, { "id": "geometric-characterisation-and-structured-2026", "type": "paper", "title": "Geometric Characterisation and Structured Trajectory Surrogates for Clinical Dataset Condensation", "short": "BTM", "arxiv": "2604.21638", "url": null, "date": "2026-04-23", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Pafue Christy Nganjimi", "Andrew Soltan", "Danielle Belgrave", "Lei Clifton", "David Clifton", "Anshul Thakur" ], "kind": "application", "settings": [ "other-data" ], "families": [ "trajectory-matching" ], "applications": [ "medical" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Gives a geometric account of why trajectory matching is hard to supervise with a small fixed synthetic set: such a set can only reproduce a limited, low-rank span of the parameter changes real SGD training induces, so a spectrally broad supervision signal creates a representability bottleneck. Proposes Bezier Trajectory Matching (BTM), which replaces SGD expert trajectories with quadratic Bezier surrogates between initial and final model states, reducing trajectory storage and better matching what a fixed synthetic set can represent. Reports BTM matching or improving on standard trajectory matching across five clinical tabular-EHR and time-series datasets (three NHS emergency-department cohorts, eICU and MIMIC-III), with the largest gains at low prevalence and low synthetic-data budgets.", "abstract": "Dataset condensation constructs compact synthetic datasets that retain the training utility of large real-world datasets, enabling efficient model development and potentially supporting downstream research in governed domains such as healthcare. Trajectory matching (TM) is a widely used condensation approach that supervises synthetic data using changes in model parameters observed during training on real data, yet the structure of this supervision signal remains poorly understood. In this paper, we provide a geometric characterisation of trajectory matching, showing that a fixed synthetic dataset can only reproduce a limited span of such training-induced parameter changes. When the resulting supervision signal is spectrally broad, this creates a conditional representability bottleneck. Motivated by this mismatch, we propose Bezier Trajectory Matching (BTM), which replaces SGD trajectories with quadratic Bezier trajectory surrogates between initial and final model states. These surrogates are optimised to reduce average loss along the path while replacing broad SGD-derived supervision with a more structured, lower-rank signal that is better aligned with the optimisation constraints of a fixed synthetic dataset, and they substantially reduce trajectory storage. Experiments on five clinical datasets demonstrate that BTM consistently matches or improves upon standard trajectory matching, with the largest gains in low-prevalence and low-synthetic-budget settings. These results indicate that effective trajectory matching depends on structuring the supervision signal rather than reproducing stochastic optimisation paths.", "notes": "Data is tabular clinical/EHR and time-series (confirmed from the paper's full text: NHS Oxford/Portsmouth/Birmingham, eICU, MIMIC-III), not images; mapped to the other-data setting." }, { "id": "soft-label-pruning-and-2026", "type": "paper", "title": "Soft Label Pruning and Quantization for Large-Scale Dataset Distillation", "arxiv": "2604.18135", "url": null, "date": "2026-04-20", "venue": { "name": "TPAMI 2026", "status": "journal" }, "authors": [ "Xiao Lingao", "Yang He" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "label-distillation" ], "tier": "notable", "links": { "code": "https://github.com/he-y/soft-label-pruning-quantization-for-dataset-distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Extends the within-class-batching idea for reducing soft-label storage with two further mechanisms: Label Pruning with Dynamic Knowledge Reuse improves label diversity per augmentation, and Label Quantization with Calibrated Student-Teacher Alignment improves diversity per image; reduces soft-label storage by 78x on ImageNet-1K and 500x on ImageNet-21K while improving accuracy by up to 7.2 and 2.8 points respectively.", "abstract": "Large-scale dataset distillation requires storing auxiliary soft labels that can be 30-40x larger on ImageNet-1K and 200x larger on ImageNet-21K than the condensed images, undermining the goal of dataset compression. We identify two fundamental issues necessitating such extensive labels: (1) insufficient image diversity, where high within-class similarity in synthetic images requires extensive augmentation, and (2) insufficient supervision diversity, where limited variety in supervisory signals during training leads to performance degradation at high compression rates. To address these challenges, we propose Label Pruning and Quantization for Large-scale Distillation (LPQLD). We enhance image diversity via class-wise batching and batch-normalization supervision during synthesis. For supervision diversity, we introduce Label Pruning with Dynamic Knowledge Reuse to improve label-per-augmentation diversity, and Label Quantization with Calibrated Student-Teacher Alignment to improve augmentation-per-image diversity. Our approach reduces soft label storage by 78x on ImageNet-1K and 500x on ImageNet-21K while improving accuracy by up to 7.2% and 2.8%, respectively. Extensive experiments validate the superiority of LPQLD across different network architectures and dataset distillation methods. Code is available at https://github.com/he-y/soft-label-pruning-quantization-for-dataset-distillation.", "builds_on": [ "lpld-2024" ], "awesome_sections": [ "Main / Label Distillation" ] }, { "id": "hard-truths-soft-labels-2026", "type": "paper", "title": "Rethinking Dataset Distillation: Hard Truths about Soft Labels", "short": "Hard truths about soft labels", "arxiv": "2604.18811", "url": null, "date": "2026-04-20", "venue": { "name": "CVPR 2026", "status": "conference" }, "authors": [ "Priyam Dey", "Aditya Sahdev", "Sunny Bhati", "Konda Reddy Mopuri", "R. Venkatesh Babu" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [], "tier": "notable", "links": { "code": null, "project": null }, "facets": { "labels": "mixed", "regime": "decoupled", "prior": [ "pretrained-classifier", "trained-experts" ], "scale": [ "imagenet-1k", "tiny-imagenet" ] }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "label-worth-2024 (Qin et al.) showed that random real images with teacher soft labels match SRe2L's reported ImageNet-1K accuracy, undercutting the claim that synthesized images are doing the work. Meanwhile the coreset literature (EL2N/GraNd, Dyn-Unc, forgetting, k-center, GraphCut, Cal) consistently shows high-quality subsets beat random under hard labels, and SelMatch's sliding-window search finds the IPC-optimal difficulty band at real training cost. Small-scale trajectory-matching methods (TM, DATM) report large gains over coresets on TinyImageNet, and DATM further jointly learns soft labels with images. No prior work had asked whether the \"coresets beat random\" finding and the \"random beats distillation\" finding could both be true at once, under different label regimes.", "problem": "Coreset selection and dataset distillation are almost never evaluated under the same label regime (coresets: hard labels; large-scale DD: cutmix-augmented per-crop teacher relabeling every epoch, termed SL+KD; some small-scale DD: one frozen soft label per image, termed SL), so it is unknown whether data quality itself stops mattering as label supervision grows, or whether DD methods are simply worse at building coresets than the coreset literature's own methods.", "idea": "Split label supervision into three regimes — hard labels (HL), one frozen soft label per image (SL), and per-crop teacher relabeling every epoch (SL+KD) — and separately scale subset quality, subset size (IPC) and training compute within each. Data quality drives performance in HL exactly as the coreset literature reports; it plays only a minor role in SL beyond a small minimum IPC, because per-sample training signal homogenizes; and in SL+KD it stops mattering almost entirely, with performance instead set by compute and saturating near full-dataset accuracy regardless of which subset (of any size or quality) is used to start.", "method": "Constructs subsets by retaining a fraction $f\\in[20\\%,100\\%]$ of ImageNet-1K by EL2N difficulty score at IPC 10-700 (class-balance capped at 700), and trains ResNet-18 students under each of HL, SL and SL+KD at four compute budgets (2-50 full-dataset-equivalent epochs), comparing to a full-dataset-trained reference at matched compute. Derives EL2N-SL, a soft-label generalization of the EL2N/GraNd score ($\\mathrm{EL2N\\text{-}SL}(x)=\\frac1T\\,\\mathbb{E}\\|p(w_t,x)-q(w_t,x)\\|_2$ against the teacher's soft target $q$) to check whether per-sample difficulty still varies under frozen soft labels. Introduces Distillation Correlation Score (DCS): given $m$ precomputed subsets of varying quality/IPC, DCS is the Spearman correlation between each subset's downstream test loss (trained and evaluated once, reused as a lookup table) and its distillation-objective loss (e.g. TM's trajectory-matching loss), letting a synthesis objective's scalability be checked without running synthesis or student training. Introduces CAD-Prune: trains one model on the full dataset under the same compute budget/LR schedule as the downstream task, then scores each sample by the standard deviation of its EL2N score over a small window of $J{=}6$ epochs positioned $W{=}2$ epochs from the end of that compute-matched run (capturing samples still being learned at that budget, not early-training difficulty), and selects the difficulty band this implies. CA2D applies CAD-Prune's selected samples to RDED's crop/select/stitch construction instead of RDED's own confidence-only patch heuristic. In the loop: a pretrained ResNet-18 teacher (for soft labels and EL2N/CAD-Prune scores); no bi-level optimization for the paper's own analysis, though it re-runs TM/DATM/DC/DM/SRe2L/DWA/D4M/Minimax Diffusion/RDED as objects of study.", "evidence": "ImageNet-1K, ResNet-18, IPC 10/50/100, same subsets scored under all three label regimes (Table 1) — the controlled comparison the atlas tracks: in HL, DD methods vary from far below random real images (SRe2L 9.79%/14.89% vs. random real 28.52%/40.89% at IPC50/100) to above it (RDED 14.34/38.49/44.36 vs. random real 5.10/28.52/40.89 at IPC10/50/100), and the best coreset (EL2N- Best, IPC-dependent difficulty window) reaches 47.18% at IPC100 — beating RDED's 44.36%; full dataset at matched compute is 65.91%. In SL (one frozen soft label per image, same images), all methods compress toward each other and toward random real (SRe2L 37.33 vs. random real 55.65 at IPC100) but a visible method gap remains. In SL+KD (per-crop relabel+KD, same images again), every method and every coreset lands within a narrow 52-59% band at IPC50 regardless of method identity or IPC (e.g. SRe2L 52.58, D4M 58.08, random real 58.10, EL2N-Best 59.16), against a full-dataset reference of 62.50% at matched compute — i.e. label regime alone, not subset choice, explains most of the score once KD is added. TinyImageNet/ConvNet-D4 (Table 2): DATM beats k-center by about 8 points under HL (29.51% vs. 22.02% at IPC50) but the gap vanishes under SL (37.39% vs. 37.45%). CAD-Prune vs. the sliding-window coreset baseline and CA2D vs. RDED, ImageNet-1K HL (Table 3): CAD-Prune reaches 40.21%/47.40% (IPC50/100) vs. sliding-window EL2N-Best's 38.44%/47.76% at 2-4x less compute (no repeated full training runs per candidate window); CA2D reaches 41.72%/46.32% vs. RDED's 38.49%/44.36%.", "limitations": "Restricted to image classification (stated as future work: multimodal, tasks beyond classification). TM/DATM's DCS-based conclusion about ResNet-18 non-scalability is inferred from surrogate-loss correlation, not from actually training a competitive TM/DATM ImageNet-1K set (TM/ DATM synthesis itself never scales past TinyImageNet/ConvNet-D4 in the paper). CAD-Prune still requires one full compute-matched training run on the whole dataset to obtain its uncertainty scores, so its efficiency gain is relative to the sliding-window search, not free. The paper states that \"expressing non-optimizable DD objectives for evaluation under DCS requires care and is an open problem.\" No wall-clock/GPU-hour costs are reported anywhere in the paper." }, "summary": "Shows that with abundant or fixed soft labels from a pretrained teacher, coresets fail to convincingly beat random subsets and performance saturates near the full-dataset level regardless of subset quality — unlike the hard-label coreset literature — and that in the hard-label setting only RDED reliably beats random baselines on ImageNet-1K among nine distillation methods tested; introduces a compute-aware pruning metric (CAD-Prune) and a resulting method (CA2D) that outperforms existing distillation methods on ImageNet-1K under a fixed compute budget.", "abstract": "Despite the perceived success of large-scale dataset distillation (DD) methods, recent evidence finds that simple random image baselines perform on-par with state-of-theart DD methods like SRe2L due to the use of soft labels during downstream model training. This is in contrast with the findings in coreset literature, where high-quality coresets consistently outperform random subsets in the hardlabel (HL) setting. To understand this discrepancy, we perform a detailed scalability analysis to examine the role of data quality under different label regimes, ranging from abundant soft labels (termed as SL+KD regime) to fixed soft labels (SL) and hard labels (HL). Our analysis reveals that high-quality coresets fail to convincingly outperform the random baseline in both SL and SL+KD regimes. In the SL+KD setting, performance further approaches nearoptimal levels relative to the full dataset, regardless of subset size or quality, for a given compute budget. This performance saturation calls into question the widespread practice of using soft labels for model evaluation, where unlike the HL setting, subset quality has negligible influence. A subsequent systematic evaluation of five large-scale and four small-scale DD methods in the HL setting reveals that only RDED reliably outperforms random baselines on ImageNet-1K, but can still lag behind strong coreset methods due to its over-reliance on easy sample patches. Based on this, we introduce CAD-Prune, a compute-aware pruning metric that efficiently identifies samples of optimal difficulty for a given compute budget, and use it to develop CA2D, a compute-aligned DD method, outperforming current DD methods on ImageNet-1K at various IPC settings. Together, our findings uncover many insights into current DD research and establish useful tools to advance dataefficient learning for both coresets and DD.", "awesome_sections": [ "Main / Better Understanding" ], "notes": "Kind kept as analysis-theory for its critique of soft-label evaluation; CA2D's own family (likely decoupled-synthesis, given ImageNet-1K/RDED comparison) is not stated explicitly in the abstract — flagged for review." }, { "id": "hopa-2026", "type": "paper", "title": "Omnimodal Dataset Distillation via High-order Proxy Alignment", "short": "HoPA", "arxiv": "2604.10666", "url": null, "date": "2026-04-12", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Yuxuan Gao", "Xiaohao Liu", "Xiaobo Xia", "Tongliang Liu" ], "kind": "method", "settings": [ "audio-visual-omni" ], "families": [ "trajectory-matching" ], "facets": { "labels": "similarity-matrix", "prior": [ "trained-experts", "pretrained-classifier" ], "regime": "bilevel", "parameterization": [ "embedding-features", "similarity-matrix" ], "scale": [ "large-scale-other" ] }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-14", "source": "arxiv-api", "depth": "full-text" }, "summary": "Extends dataset distillation beyond two modalities to omnimodal settings by identifying the factor that bounds endpoint discrepancy as the number of modalities grows, then aligns modalities through a compact proxy that captures high-order cross-modal structure directly instead of modeling every pairwise combination; the proxy is compatible with trajectory matching. Reports better compression-performance trade-offs than bimodal distillation baselines across several omnimodal benchmarks, supported by a spectral-perspective theoretical analysis.", "abstract": "Dataset distillation compresses large-scale datasets into compact synthetic sets while preserving training performance, but existing methods are largely restricted to single-modal or bimodal settings. Extending dataset distillation to scenarios involving more than two modalities, i.e., Omnimodal Dataset Distillation, remains underexplored and challenging due to increased heterogeneity and complex cross-modal interactions. In this work, we identify the key determinant that bounds the endpoint discrepancy in the omnimodal setting, which is exacerbated with an increasing number of modalities. To this end, we propose HoPA, a unified method that captures high-order cross-modal alignments via a compact proxy, which is compatible with trajectory matching as well. By abstracting omnimodal alignment with a shared similarity structure, our method avoids the combinatorial complexity of pairwise modality modeling and enables scalable joint distillation across heterogeneous modalities. Theoretical analysis from the spectral perspective reveals the rationality of our proposed method against bimodal dataset distillation techniques. Extensive experiments on various benchmarks demonstrate that the proposed method achieves superior compression-performance trade-offs compared to existing competitors. The source code will be publicly released.", "builds_on": [ "lors-2024", "repblend-2025", "imagebinddc-2026" ], "explained": { "before": "Bimodal trajectory-matching methods (LoRS, RepBlend) handle exactly two modalities, matching trajectories and a learned pairwise similarity structure. ImageBindDC condenses in a shared (ImageBind) embedding space but via distribution matching, not trajectory matching, and does not analyze how many modalities its objective can scale to.", "problem": "The paper proves (their Lemma 1) that the trajectory-matching endpoint-discrepancy bound sums one mismatch term per modality, so going from $k{=}2$ to more modalities adds more terms to the bound without bounding the interaction *between* modalities. Naively extending bimodal pairwise trajectory matching to $k$ modalities by modeling every pair requires $\\mathcal O(k^2)$ pairwise objectives and produces similarity tensors of size $\\mathcal O(k^2N^2)$, which the paper calls computationally prohibitive and poorly scalable as more modalities are added.", "idea": "Instead of modeling every modality pair, extract a single scalar \"unified semantic feature\" per instance - the leading singular vector of the SVD of the per-instance cross-modality Gram matrix - and use it as a compact proxy for the instance's full high-order cross-modal structure; this proxy slots into trajectory matching with minimal changes to the bimodal recipe (LoRS-style trajectory matching plus a learned similarity matrix) while avoiding the combinatorial pairwise blow-up.", "method": "Synthetic per-modality embeddings and a learnable similarity matrix $S_e$ are optimized via bi-level trajectory matching: $\\mathcal L_{omni\\text{-}traj}=\\sum_{m\\in\\mathcal M}\\|\\theta^t_{e,m}-\\theta_{T,m}\\|_2^2/\\|\\theta_{0,m}-\\theta_{T,m}\\|_2^2$, matching expert trajectories (10 epochs of real-data training, 20 independent runs) per modality-specific projection head. A pretrained ImageBind encoder backbone is frozen throughout (only the projection heads are trained/matched). For each instance, a Gram matrix $G=zz^\\top$ of its stacked modality embeddings is SVD-decomposed; by the Eckart-Young theorem (their Lemma 2) the rank-1 approximation $\\tilde G=\\sigma_1^2u_1u_1^\\top$ is the optimal single-vector summary, and its leading singular vector $v_1$ (with $\\sigma_1v_1=\\sum_{m}[u_1]_m z_m$) becomes the \"compact proxy\" - instance-to-instance similarity is then approximated as $\\tilde s(x_i,x_j)=v_1^{(i)\\top}v_1^{(j)}$, a single scalar instead of $\\mathcal O(k^2)$ pairwise terms. The total loss adds a modality-level contrastive term on the leading singular value ($\\mathcal L_M=-\\frac1N\\sum_i\\log\\frac{\\exp(\\sigma_1^{(i)}/\\tau)}{\\sum_\\ell\\exp(\\sigma_\\ell^{(i)}/ \\tau)}$, pulling the top singular value above the rest) to a weighted-BCE term enforcing consistency between $S_e$ and the proxy similarities. A spectral-perspective theorem (their Theorem 1) shows a single-mode (rank-1) objective has zero spectral weight on non-principal modes, while a full-spectrum pairwise objective does not, and that the single-mode per-step mismatch bound is dominated by (i.e. no worse than) the full-spectrum bound whenever non-principal modes carry error - the paper's formal argument for why the compact proxy does not lose the alignment that matters.", "evidence": "Three omnimodal (video+audio+text) benchmarks: MSR-VTT, VGGSound-S, DiDeMo, evaluated by average R@1/R@5 retrieval. MSR-VTT@500 (Table 1): HoPA R@1/R@5 22.1/42.9 vs LoRS 21.8/42.4, vs full-data 22.6/44.8. VGGSound-S@500 (Table 2): HoPA 13.4/36.4 vs LoRS 13.3/35.6, vs full-data 13.7/37.7. DiDeMo@100 (Table 3): HoPA 18.4/37.2 vs LoRS (at 500, 5x the budget) 17.3/37.6 - HoPA beats LoRS's larger-budget number with a fifth of the pairs, the paper's strongest compression claim. Cost: despite added SVD overhead, HoPA distillation wall-clock is roughly half of LoRS's at comparable budgets (e.g. MSR-VTT@500: HoPA 57.6 min vs LoRS 120.4 min) because it distills all modalities jointly rather than pairwise. Cross-architecture transfer (Table 4): data distilled on ImageBind transfers well to OmniBind (R@1 13.13 vs ImageBind-source 13.15) but poorly to LanguageBind (R@1 5.93), a large drop. The key ablation (Table 5, MSR-VTT@500) compares the omniwise (joint) proxy against three pairwise variants: HoPA 22.25% vs a 3-pair baseline 18.11%, a text-anchored (\"T-bind\") variant 20.19%, and a vision-anchored (\"V-bind\") variant 19.85%, with pairwise variants degrading further as the budget grows while HoPA improves. A rank ablation (Table 6) shows rank-1 beats rank-2 (MSR-VTT R@1 22.25% vs 20.48%), supporting the single-leading-singular-vector design choice over a richer multi-mode proxy. An SVD-cost ablation (Table 7) reports the SVD/backprop-through-SVD overhead as 3.5% extra cost during expert-trajectory (buffer) generation but 39.1% extra during the distillation step itself, due to higher-order derivatives through the SVD.", "limitations": "The paper states (Appendix F) that the underlying TESLA-style trajectory-matching algorithm has a non-trivial memory footprint that limits scalability to very large synthetic datasets. The 39.1% per-iteration overhead of differentiating through the SVD could become a bottleneck at higher embedding dimensions or larger modality counts $k$. The method assumes frozen pretrained encoders throughout and does not explore end-to-end fine-tuning. All experiments use exactly three modalities (video, audio, text); generalization to four or more modalities simultaneously, which motivates the paper's framing, is not empirically tested. Cross-architecture transfer to a different pretrained joint-embedding backbone (LanguageBind) shows a large accuracy drop relative to transferring to a closely related one (OmniBind), suggesting the proxy's benefit is tied to the specific embedding geometry it was distilled in." }, "notes": "" }, { "id": "learnability-gd-2026", "type": "paper", "title": "Learnability-Guided Diffusion for Dataset Distillation", "short": "Learnability-guided diffusion", "arxiv": "2604.00519", "url": null, "date": "2026-04-01", "venue": { "name": "CVPR 2026", "status": "conference" }, "authors": [ "Jeffrey A. Chan-Santiago", "Mubarak Shah" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "prior": [ "diffusion" ], "regime": "training-free", "parameterization": [ "pixels" ], "scale": [ "imagenet-subsets", "imagenet-1k" ] }, "tier": "notable", "links": { "code": null, "project": "https://jachansantiago.com/learnability-guided-distillation/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Finds that prior diffusion-based distillation methods produce redundant samples -- disjoint subsets of a distilled set share 80-90% overlapping signal -- because they optimize visual diversity or average training dynamics without accounting for sample-to-sample similarity; Learnability-Guided Diffusion instead builds the synthetic set incrementally, training a model on what exists so far and generating new samples guided by learnability scores balanced against validity under a reference model, cutting redundancy by 39.1% and reaching 60.1% on ImageNet-1K, 87.2% on ImageNette and 72.9% on ImageWoof.", "abstract": "Training machine learning models on massive datasets is expensive and time-consuming. Dataset distillation addresses this by creating a small synthetic dataset that achieves the same performance as the full dataset. Recent methods use diffusion models to generate distilled data, either by promoting diversity or matching training gradients. However, existing approaches produce redundant training signals, where samples convey overlapping information. Empirically, disjoint subsets of distilled datasets capture 80-90% overlapping signals. This redundancy stems from optimizing visual diversity or average training dynamics without accounting for similarity across samples, leading to datasets where multiple samples share similar information rather than complementary knowledge. We propose learnability-driven dataset distillation, which constructs synthetic datasets incrementally through successive stages. Starting from a small set, we train a model and generate new samples guided by learnability scores that identify what the current model can learn from, creating an adaptive curriculum. We introduce Learnability-Guided Diffusion (LGD), which balances training utility for the current model with validity under a reference model to generate curriculum-aligned samples. Our approach reduces redundancy by 39.1%, promotes specialization across training stages, and achieves state-of-the-art results on ImageNet-1K (60.1%), ImageNette (87.2%), and ImageWoof (72.9%). Our code is available on our project page https://jachansantiago.github.io/learnability-guided-distillation/.", "builds_on": [ "minimax-2024", "mgd3-2025" ], "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "explained": { "before": "Minimax fine-tunes for representativeness/diversity, MGD3 guides toward discovered modes, and IGD (Influence-Guided Diffusion, its closest direct predecessor here) steers sampling to match training gradients from the full dataset -- but all three synthesize every sample of a class under one static objective (a fixed diversity criterion or a fixed average-gradient target).", "problem": "Optimizing every sample toward the same static target (visual diversity, or the average training gradient) makes samples converge to similar information content: the paper's own cross-validation experiment shows a model trained on any one 10-IPC increment of a 50-IPC set already gets 80-90% (average 94.7% for raw DiT, 87.1% for IGD) of the accuracy achievable from a disjoint increment, meaning the \"extra\" samples carry little additional learning signal. The root cause identified: a single sample cannot simultaneously suit early training (which needs strong, coarse-feature gradients) and late training (small, fine-grained gradients), so optimizing for \"the average\" produces samples useful at no specific stage.", "idea": "Stop treating distillation as a single-shot synthesis problem; build the distilled set incrementally, training a model on each increment before generating the next, and condition new samples on a \"learnability score\" that rewards what the current model still struggles with while penalizing samples a reference model (trained on the full dataset) finds invalid or unrepresentative -- so each new increment targets exactly the current model's learning frontier instead of repeating earlier signal.", "method": "The final set of size $M$ is built as $K$ disjoint increments $\\mathcal{I}_1,...,\\mathcal{I}_K$; stage $i$ trains a learner $\\theta_{i-1}$ on the cumulative set $\\mathcal{D}_{i-1}=\\bigcup_{k0$ tuned per stage (coarse/mid/fine amplification schedule, e.g. 5-5-0.5). Only pixels are stored; the fine-tuned VAR and its class tokens are not part of the distilled artifact.", "evidence": "ImageNet-1K, hard-label, ResNet-18/ResNet-101 (Table 1): IPC10 47.6% / 52.1% vs CaO2 46.1% and RDED 44.3% (ResNet-18); IPC50 60.8% / 66.4% vs CaO2 60.0%; IPC100 62.7% (matching CaO2's 62.7%). ImageWoof IPC10, ResNet-18: 45.8% vs RDED 40.1%, D3HR 39.6%. CIFAR-10 IPC10, ResNet-18: 44.3% vs D3HR 41.3%, RDED 37.1%. FID at IPC10 (Table 6): 17.3+/-0.1 vs Minimax 18.3+/-0.2, D3HR 19.0+/-0.2 (lower is better). Only Minimax, D3HR, RDED and CaO2 are compared; no D4M, MGD3 or IGD comparison anywhere, even in related work. Cost (Tables 8-9): inference 0.147+/-0.001 s/image vs D3HR's DDIM-based 0.456+/-0.002 s/image; peak memory 1.840+/-0.001 GB, marginal overhead over the unmodified VAR's 1.770 GB. Ablation (Table 3): 5 amplification steps at 50% top-attention across all stages is the best single configuration (47.6+/-0.1%); Table 4 shows a balanced coarse-mid-fine schedule (5-5-0.5) needed, and coarse-stage amplification contributes the largest single share of the gain.", "limitations": "Stated: no dedicated Limitations section (the Conclusion only states the method's achievements). Observed: requires fine-tuning the VAR generator per dataset (5 epochs with a class-token head added), so it depends on the same target-trained-generator pattern the training-free branch of diffusion-guided synthesis is trying to remove; only compared against Minimax, D3HR, RDED and CaO2, omitting D4M, MGD3, CoDA, DAP, IGD and the entire training-free/VLM-conditioned diffusion literature, so its standing against the field's strongest diffusion baselines is untested; label regime is described as hard-label evaluation though the RDED protocol it follows uses a ResNet-18 teacher, leaving the degree of teacher-relabeling ambiguous in the paper's own framing; amplification strengths $\\beta_n$ and $\\rho$ are tuned per stage without a stated cross-dataset transfer study." } }, { "id": "vq-soft-labels-2026", "type": "paper", "title": "Vector-Quantized Soft Label Compression for Dataset Distillation", "short": "VQAE", "arxiv": "2603.03808", "url": null, "date": "2026-03-04", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Ali Abbasi", "Ashkan Shahbazi", "Hamed Pirsiavash", "Soheil Kolouri" ], "kind": "method", "settings": [ "image-classification", "text" ], "families": [ "label-distillation" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Quantifies the bit cost of soft teacher labels across augmentations in dataset distillation, showing they dominate total storage in large-class settings such as ImageNet-1K, and introduces a vector-quantized autoencoder (VQAE) to compress them. Reports 30-40x additional compression over RDED, LPLD, SRe2L and CDA on ImageNet-1K while retaining over 90% of their original accuracy, and validates the approach on both vision and language distillation benchmarks.", "abstract": "Dataset distillation is an emerging technique for reducing the computational and storage costs of training machine learning models by synthesizing a small, informative subset of data that captures the essential characteristics of a much larger dataset. Recent methods pair synthetic samples and their augmentations with soft labels from a teacher model, enabling student models to generalize effectively despite the small size of the distilled dataset. While soft labels are critical for effective distillation, the storage and communication overhead they incur, especially when accounting for augmentations, is often overlooked. In practice, each distilled sample is associated with multiple soft labels, making them the dominant contributor to storage costs, particularly in large-class settings such as ImageNet-1K. In this paper, we present a rigorous analysis of bit requirements across dataset distillation frameworks, quantifying the storage demands of both distilled samples and their soft labels. To address the overhead, we introduce a vector-quantized autoencoder (VQAE) for compressing soft labels, achieving substantial compression while preserving the effectiveness of the distilled data. We validate our method on both vision and language distillation benchmarks. On ImageNet-1K, our proposed VQAE achieves 30--40x additional compression over RDED, LPLD, SRE2L, and CDA baselines while retaining over $90\\%$ of their original performance.", "notes": "" }, { "id": "unirain-2026", "type": "paper", "title": "UniRain: Unified Image Deraining with RAG-based Dataset Distillation and Multi-objective Reweighted Optimization", "short": "UniRain", "arxiv": "2603.03967", "url": null, "date": "2026-03-04", "venue": { "name": "CVPR 2026", "status": "conference" }, "authors": [ "Qianfeng Yang", "Qiyuan Guan", "Xiang Chen", "Jiyu Jin", "Guiyue Jin", "Jiangxin Dong" ], "kind": "application", "settings": [ "dense-prediction" ], "families": [ "coreset-selection" ], "applications": [ "restoration" ], "tier": "catalogued", "links": { "code": "https://github.com/QianfengY/UniRain", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "UniRain is a unified image-deraining framework that trains one asymmetric mixture-of-experts model across rain streak/raindrop and day/night conditions; to build its mixed training set it uses a retrieval-augmented-generation-based dataset distillation pipeline that selects high-quality samples from all public deraining datasets, alongside a multi-objective reweighted optimization strategy. The paper reports favorable performance against state-of-the-art deraining models on its proposed benchmarks and multiple public datasets.", "abstract": "Despite significant progress has been made in image deraining, we note that most existing methods are often developed for only specific types of rain degradation and fail to generalize across diverse real-world rainy scenes. How to effectively model different rain degradations within a universal framework is important for real-world image deraining. In this paper, we propose UniRain, an effective unified image deraining framework capable of restoring images degraded by rain streak and raindrop under both daytime and nighttime conditions. To better enhance unified model generalization, we construct an intelligent retrieval augmented generation (RAG)-based dataset distillation pipeline that selects high-quality training samples from all public deraining datasets for better mixed training. Furthermore, we incorporate a simple yet effective multi-objective reweighted optimization strategy into the asymmetric mixture-of-experts (MoE) architecture to facilitate consistent performance and improve robustness across diverse scenes. Extensive experiments show that our framework performs favorably against the state-of-the-art models on our proposed benchmarks and multiple public datasets.", "notes": "Borderline scope: \"dataset distillation\" here names a RAG-based sample-selection step inside a larger unified deraining system, not a standalone condensation method; classified as application + coreset-selection (selection-based, not synthesis) rather than rejected, since the atlas does include selection baselines. Flagged in the handoff for a second opinion." }, { "id": "harmonic-dataset-distillation-for-2026", "type": "paper", "title": "Harmonic Dataset Distillation for Time Series Forecasting", "arxiv": "2603.03760", "url": null, "date": "2026-03-04", "venue": { "name": "AAAI 2026", "status": "conference" }, "authors": [ "Seungha Hong", "Sanghwan Jang", "Wonbin Kweon", "Suyeon Kim", "Gyuseok Lee", "Hwanjo Yu" ], "kind": "method", "settings": [ "time-series" ], "families": [ "distribution-matching" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "HDT (Harmonic Dataset Distillation) decomposes each time series into its sinusoidal basis via the FFT and condenses by Harmonic Matching, aligning the core periodic structure of the synthetic and real series in the frequency domain so that updates never disturb temporal dependencies. The paper reports strong cross-architecture generalization and scalability for time series forecasting compared to conventional, image-centric condensation methods adapted to time series.", "abstract": "Time Series forecasting (TSF) in the modern era faces significant computational and storage cost challenges due to the massive scale of real-world data. Dataset Distillation (DD), a paradigm that synthesizes a small, compact dataset to achieve training performance comparable to that of the original dataset, has emerged as a promising solution. However, conventional DD methods are not tailored for time series and suffer from architectural overfitting and limited scalability. To address these issues, we propose Harmonic Dataset Distillation for Time Series Forecasting (HDT). HDT decomposes the time series into its sinusoidal basis through the FFT and aligns the core periodic structure by Harmonic Matching. Since this process operates in the frequency domain, all updates during distillation are applied globally without disrupting temporal dependencies of time series. Extensive experiments demonstrate that HDT achieves strong cross-architecture generalization and scalability, validating its practicality for large-scale, real-world applications.", "notes": "Family assignment (distribution-matching) is a best fit for frequency-domain statistic alignment; the abstract does not name gradient/trajectory/distribution matching explicitly." }, { "id": "fixed-anchors-are-not-2026", "type": "paper", "title": "Fixed Anchors Are Not Enough: Dynamic Retrieval and Persistent Homology for Dataset Distillation", "arxiv": "2602.24144", "url": null, "date": "2026-02-27", "venue": { "name": "CVPR 2026", "status": "conference" }, "authors": [ "Muquan Li", "Hang Gou", "Yingyi Ma", "Rongzheng Wang", "Ke Qin", "Tao He" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Identifies a 'pull-to-anchor' effect in residual-matching decoupled pipelines that rely on static real patches, which reduces intra-class diversity; RETA's Dynamic Retrieval Connection selects a real patch from a prebuilt pool by minimizing a fit-complexity score in teacher feature space before injecting it via a residual connection, and Persistent Topology Alignment penalizes topological discrepancies between real and synthetic sets via persistence images over a mutual k-NN feature graph; reports 64.3% top-1 on ImageNet-1K with ResNet-18 at 50 images per class, +3.1 points over the prior best.", "abstract": "Decoupled dataset distillation (DD) compresses large corpora into a few synthetic images by matching a frozen teacher's statistics. However, current residual-matching pipelines rely on static real patches, creating a fit-complexity gap and a pull-to-anchor effect that reduce intra-class diversity and hurt generalization. To address these issues, we introduce RETA -- a Retrieval and Topology Alignment framework for decoupled DD. First, Dynamic Retrieval Connection (DRC) selects a real patch from a prebuilt pool by minimizing a fit-complexity score in teacher feature space; the chosen patch is injected via a residual connection to tighten feature fit while controlling injected complexity. Second, Persistent Topology Alignment (PTA) regularizes synthesis with persistent homology: we build a mutual k-NN feature graph, compute persistence images of components and loops, and penalize topology discrepancies between real and synthetic sets, mitigating pull-to-anchor effect. Across CIFAR-100, Tiny-ImageNet, ImageNet-1K, and multiple ImageNet subsets, RETA consistently outperforms various baselines under comparable time and memory, especially reaching 64.3% top-1 accuracy on ImageNet-1K with ResNet-18 at 50 images per class, +3.1% over the best prior.", "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "manifoldgd-2026", "type": "paper", "title": "ManifoldGD: Training-Free Hierarchical Manifold Guidance for Diffusion-Based Dataset Distillation", "short": "ManifoldGD", "arxiv": "2602.23295", "url": null, "date": "2026-02-26", "venue": { "name": "CVPR 2026", "status": "conference" }, "authors": [ "Ayush Roy", "Wei-Yang Alex Lee", "Rudrasis Chakraborty", "Vishnu Suresh Lokhande" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "prior": [ "diffusion" ], "regime": "training-free", "parameterization": [ "pixels" ], "scale": [ "imagenet-subsets", "imagenet-1k" ] }, "tier": "notable", "links": { "code": "https://github.com/AyushRoy2001/ManifoldGD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Argues that existing training-free diffusion guidance for distillation relies on simple mode-based centroids that are suboptimal, and instead computes a multi-scale coreset of prototypes via hierarchical divisive clustering of VAE latent features, then at every denoising step projects the mode-alignment vector onto the local tangent space of a manifold built from nearby prototypes, keeping generation manifold-faithful while staying semantically consistent; the first geometry-aware training-free distillation framework, improving FID, real-synthetic embedding distance and classification accuracy over prior training-free and training-based baselines.", "abstract": "In recent times, large datasets hinder efficient model training while also containing redundant concepts. Dataset distillation aims to synthesize compact datasets that preserve the knowledge of large-scale training sets while drastically reducing storage and computation. Recent advances in diffusion models have enabled training-free distillation by leveraging pre-trained generative priors; however, existing guidance strategies remain limited. Current score-based methods either perform unguided denoising or rely on simple mode-based guidance toward instance prototype centroids (IPC centroids), which often are rudimentary and suboptimal. We propose Manifold-Guided Distillation (ManifoldGD), a training-free diffusion-based framework that integrates manifold consistent guidance at every denoising timestep. Our method employs IPCs computed via a hierarchical, divisive clustering of VAE latent features, yielding a multi-scale coreset of IPCs that captures both coarse semantic modes and fine intra-class variability. Using a local neighborhood of the extracted IPC centroids, we create the latent manifold for each diffusion denoising timestep. At each denoising step, we project the mode-alignment vector onto the local tangent space of the estimated latent manifold, thus constraining the generation trajectory to remain manifold-faithful while preserving semantic consistency. This formulation improves representativeness, diversity, and image fidelity without requiring any model retraining. Empirical results demonstrate consistent gains over existing training-free and training-based baselines in terms of FID, l2 distance among real and synthetic dataset embeddings, and classification accuracy, establishing ManifoldGD as the first geometry-aware training-free data distillation framework.", "builds_on": [ "mgd3-2025", "minimax-2024", "d4m-2024" ], "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "explained": { "before": "MGD3 guides diffusion sampling toward K-means-derived class-mode centroids in VAE latent space using a Euclidean (Gaussian-kernel) attraction term, without any fine-tuning. Minimax and D4M represent the training-based and prototype-clustering strands respectively.", "problem": "MGD3-style \"mode guidance\" pulls the denoising trajectory toward a centroid using an ordinary Euclidean vector in ambient latent space, but the true data manifold is a curved, lower-dimensional subset of that space; the paper proves (via a tangent/normal decomposition) that this attraction generally has a nonzero normal-space component, which pushes the trajectory off the data manifold especially as $t\\to0$ when the marginal $p_t$ concentrates sharply near the manifold -- producing artifacts (e.g. a dog with legs in an unusual position, blurred textures) even though the class-level semantic direction is correct. Separately, K-means-based IPC centroids (as in D4M/MGD3) capture only one spatial scale of class structure.", "idea": "Build a multi-scale set of prototypes with hierarchical (top-down, divisive bisecting K-means) clustering of class-wise VAE latents instead of flat K-means, so centroids span both coarse and fine modes; then, at every denoising step, decompose the standard mode-guidance vector into components tangent and normal to a locally-estimated data manifold (built from the noised neighborhood of the nearest prototype) and subtract the normal component, so guidance only pushes the trajectory along directions the manifold can actually support.", "method": "Conditional score decomposes as $\\nabla_{x_t}\\log p_t(x_t|c)=\\nabla_{x_t}\\log p_t(x_t) + \\nabla_{x_t} \\log p_t(c|x_t)$ (marginal denoising + mode guidance); mode guidance is generalized to any PSD kernel potential $\\phi$ (Gaussian/RBF recovers MGD3, plus Laplace and inverse-multiquadric are tested) giving $g^t_{mode}=-\\phi'(\\|x_t-c\\|_2)\\frac{x_t-c}{\\|x_t-c\\|_2}$. IPC centroids $c_s$ come from a divisive (bisecting K-means) hierarchical clustering tree over VAE latents per class; a coarse-to-fine sweep from a chosen start depth $s_{start}$ down to the root picks one centroid per level (deeper splits for well-separated classes, shallower for overlapping ones), with any remaining IPC quota filled by random leaf centroids. For each chosen centroid's local latent neighborhood $\\mathcal{N}_s$, forward-diffuse it to timestep $t$'s noise level to get a local point cloud $\\mathcal{M}_t^{(s)}$; find the current sample $x_t$'s $K_t$ nearest neighbors in this cloud, take the empirical covariance's leading eigenvectors as the tangent-space basis, and build projector $P_{\\mathcal{N}_t}=I-P_{\\mathcal{T}_t}$; the corrected guidance is $g^t_{manifold}=g^t_{mode}-P_{\\mathcal{N}_t}g^t_{mode}$, i.e. the normal component is subtracted, not the tangent component kept exclusively (explicit normal subtraction rather than strict tangent projection, to avoid over-smoothing per Remark 1). The neighborhood radius for tangent estimation is annealed (exponential decay found best) across timesteps, and guidance is switched off after a stop step $T_{STOP}$ (tuned, best around 20-25 of 50 steps). No fine-tuning; base generator is DiT.", "evidence": "ImageNette hard-label protocol, ResNetAP-10 (Table 1), IPC10/20/50: 64.1±1.2/69.7±0.4/78.4±1.0, beating the reimplemented MGD3 61.9±0.6/66.5±0.5/77.5±1.1 and the unguided reimplemented DiT 59.1±0.7/64.8±1.2/ 73.3±0.9, and matching or exceeding several training-based methods (Minimax 62.0±0.2 at IPC10, D4M 64.8±3.6 at IPC10) under the same protocol. ImageNet-100 (Table 1), IPC10/20: 27.6±0.5/35.3±0.5 vs MGD3 26.1±0.6/33.2±0.5. Hard-label-only, full ImageNet-1k, ConvNet-6 (Table 10, appendix), IPC1/50: 3.1/21.4 vs MGD3 2.8±0.9/20.3±1.1 and DiT 2.6±0.3/18.5±1.3 -- markedly lower absolute numbers than soft-label-protocol papers (e.g. CoDA, DAP) because this is the harder hard-label-from-scratch evaluation, not directly comparable across protocol. Ablation (Table 3, ImageNette IPC10, ConvNet-6/ ResNetAP-10/ResNet-18): flat K-means 56.3/61.0/59.7 to divisive (no level-wise) 57.4/62.5/58.5 to divisive-levelwise 59.2/63.3/61.1 to full method with manifold guidance 60.5/64.0/62.3 to annealed radius 60.8/64.5/62.7 -- isolating that both the clustering upgrade and the manifold-correction term each contribute, and are complementary (agglomerative clustering alone is worst: 37.7/45.9/42.6). Kernel-choice ablation (Table 5) shows manifold guidance improves results regardless of the underlying kernel (RBF/Laplace/IMQ), evidencing the correction is a general mechanism, not tied to MGD3's specific Gaussian kernel. $T_{STOP}$ ablation (Fig. 8) shows FID and accuracy improve as guidance is applied through early-to-mid denoising up to ~step 25 of 50, then degrade if applied too late into the low-noise regime.", "limitations": "Stated (Conclusion): at high noise levels the local neighborhood used to estimate the tangent space is itself corrupted by diffusion noise, biasing the tangent/normal estimate; low-rank tangent approximations struggle on highly curved manifolds, risking over-smoothing and reduced diversity; the paper mitigates these heuristically (adaptive neighborhood size/radius, ridge-regularized covariance, annealing) but states that formal analysis of projection error and curvature sensitivity is future work. Observed: evaluated only under the hard-label protocol, so its absolute numbers (e.g. ImageNet-1k IPC50 21.4%) cannot be compared to the soft-label-protocol numbers most other 2025-2026 diffusion papers report (e.g. CoDA 60.4%, DAP 62.7%) on the same benchmark; the tangent-space estimation adds a k-nearest-neighbor search and eigendecomposition at every denoising step for every sample, whose wall-clock cost relative to unguided or MGD3 sampling is not quantified in the main text; several hyperparameters (start depth $s_{start}$, neighborhood size $K_t$, radius annealing schedule, $T_{STOP}$) are tuned per dataset." } }, { "id": "c-2-tc-a-2026", "type": "paper", "title": "C$^{2}$TC: A Training-Free Framework for Efficient Tabular Data Condensation", "short": "C2TC", "arxiv": "2602.21717", "url": null, "date": "2026-02-25", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Sijia Xu", "Fan Li", "Xiaoyang Wang", "Zhengyi Yang", "Xuemin Lin" ], "kind": "method", "settings": [ "other-data" ], "families": [ "dataset-quantization" ], "facets": { "regime": "training-free" }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Condenses tabular data without any gradient-based training by reformulating condensation as class-adaptive cluster allocation: a heuristic local search alternates between soft cluster assignment and class-wise clustering, and a hybrid categorical encoding keeps the clustering meaningful over heterogeneous discrete features. Reports at least two orders of magnitude faster condensation than prior gradient-based tabular baselines across 10 real-world datasets, with better downstream accuracy.", "abstract": "Tabular data is the primary data format in industrial relational databases, underpinning modern data analytics and decision-making. However, the increasing scale of tabular data poses significant computational and storage challenges to learning-based analytical systems. This highlights the need for data-efficient learning, which enables effective model training and generalization using substantially fewer samples. Dataset condensation (DC) has emerged as a promising data-centric paradigm that synthesizes small yet informative datasets to preserve data utility while reducing storage and training costs. However, existing DC methods are computationally intensive due to reliance on complex gradient-based optimization. Moreover, they often overlook key characteristics of tabular data, such as heterogeneous features and class imbalance. To address these limitations, we introduce C$^{2}$TC (Class-Adaptive Clustering for Tabular Condensation), the first training-free tabular dataset condensation framework that jointly optimizes class allocation and feature representation, enabling efficient and scalable condensation. Specifically, we reformulate the dataset condensation objective into a novel class-adaptive cluster allocation problem (CCAP), which eliminates costly training and integrates adaptive label allocation to handle class imbalance. To solve the NP-hard CCAP, we develop HFILS, a heuristic local search that alternates between soft allocation and class-wise clustering to efficiently obtain high-quality solutions. Moreover, a hybrid categorical feature encoding (HCFE) is proposed for semantics-preserving clustering of heterogeneous discrete attributes. Extensive experiments on 10 real-world datasets demonstrate that C$^{2}$TC improves efficiency by at least 2 orders of magnitude over state-of-the-art baselines, while achieving superior downstream performance.", "notes": "Bin-and-sample-per-cluster mechanism most resembles dataset-quantization, adapted to tabular features; no network-based matching is involved." }, { "id": "dataset-color-quantization-a-2026", "type": "paper", "title": "Dataset Color Quantization: A Training-Oriented Framework for Dataset-Level Compression", "arxiv": "2602.20650", "url": null, "date": "2026-02-24", "venue": { "name": "ICLR 2026", "status": "conference" }, "authors": [ "Chenyue Yu", "Lingao Xiao", "Jinhong Deng", "Ivor W. Tsang", "Yang He" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "dataset-quantization", "parameterization" ], "facets": { "parameterization": [ "color-quantized" ] }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Reduces color-space redundancy across a dataset by enforcing consistent palettes across similar images, selectively retaining semantically important colors under model-perception guidance, and preserving structural detail needed for feature learning; reports improved training performance under aggressive compression on CIFAR-10/100, Tiny-ImageNet and ImageNet-1K.", "abstract": "Large-scale image datasets are fundamental to deep learning, but their high storage demands pose challenges for deployment in resource-constrained environments. While existing approaches reduce dataset size by discarding samples, they often ignore the significant redundancy within each image -- particularly in the color space. To address this, we propose Dataset Color Quantization (DCQ), a unified framework that compresses visual datasets by reducing color-space redundancy while preserving information crucial for model training. DCQ achieves this by enforcing consistent palette representations across similar images, selectively retaining semantically important colors guided by model perception, and maintaining structural details necessary for effective feature learning. Extensive experiments across CIFAR-10, CIFAR-100, Tiny-ImageNet, and ImageNet-1K show that DCQ significantly improves training performance under aggressive compression, offering a scalable and robust solution for dataset-level storage reduction.", "awesome_sections": [ "Main / Dataset Quantization" ] }, { "id": "pds-2026", "type": "paper", "title": "Multimodal Dataset Distillation Made Simple by Prototype-Guided Data Synthesis", "short": "PDS", "arxiv": "2602.19756", "url": null, "date": "2026-02-23", "venue": { "name": "ICLR 2026", "status": "conference" }, "authors": [ "Junhyeok Choi", "Sangwoo Mo", "Minwoo Chae" ], "kind": "method", "settings": [ "vision-language" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "labels": "hard", "regime": "training-free", "prior": [ "diffusion", "vision-language-model" ], "parameterization": [ "pixels", "embedding-features" ], "scale": [ "flickr30k-coco" ] }, "tier": "core", "links": { "code": "https://github.com/junhyeok9712/PDS", "project": null }, "status": { "verified": true, "verified_on": "2026-09-14", "source": "arxiv-api", "depth": "full-text" }, "summary": "PDS is a learning-free multimodal distillation framework that uses CLIP to extract aligned image-text embeddings, derives prototypes from them, and synthesizes images with an unCLIP decoder, avoiding the full-dataset training and joint pixel/text optimization that architecture-dependent distillation methods require. The paper reports consistently outperforming optimization-based dataset distillation and subset-selection baselines and achieving state-of-the-art cross-architecture generalization.", "abstract": "Recent advances in multimodal learning have achieved remarkable success across diverse vision-language tasks. However, such progress heavily relies on large-scale image-text datasets, making training costly and inefficient. Prior efforts in dataset filtering and pruning attempt to mitigate this issue, but still require relatively large subsets to maintain performance and fail under very small subsets. Dataset distillation offers a promising alternative, yet existing multimodal dataset distillation methods require full-dataset training and joint optimization of image pixels and text features, making them architecture-dependent and limiting cross-architecture generalization. To overcome this, we propose a learning-free dataset distillation framework that eliminates the need for large-scale training and optimization while enhancing generalization across architectures. Our method uses CLIP to extract aligned image-text embeddings, obtains prototypes, and employs an unCLIP decoder to synthesize images, enabling efficient and scalable multimodal dataset distillation. Extensive experiments demonstrate that our approach consistently outperforms optimization-based dataset distillation and subset selection methods, achieving state-of-the-art cross-architecture generalization.", "builds_on": [ "lors-2024", "mtt-vl-2024" ], "explained": { "before": "Trajectory-matching methods (MTT-VL, LoRS) and the generative method EDGE all require either full-dataset training and bi-level optimization through the encoders (MTT-VL, LoRS) or per-dataset fine-tuning of a diffusion model (EDGE), which the paper argues makes the synthesized set architecture-dependent: because pixels are optimized adversarially against a specific backbone's gradients, the result generalizes poorly to a different evaluation architecture. Learning-free unimodal methods (D4M, MGD3) exist for image classification but do not transfer to paired data because the VAE embeddings they use are not aligned with a separate text encoder's embeddings.", "problem": "The paper diagnoses that optimization-based multimodal distillation effectively bakes architecture-specific adversarial perturbations into the synthetic images (since the optimization process differentiates through one particular image/text encoder pair), and that its parameter count and hyperparameter surface (learning rates for both pixels and text, at minimum) grow linearly with the number of pairs, making the method costlier and less transferable as budgets grow. Naively adapting unimodal learning-free synthesis fails because there is no off-the-shelf decoder that is both trained on an aligned image-text embedding space and requires no per-dataset training.", "idea": "Do dataset distillation with no gradient-based optimization of any kind: extract CLIP's already cross-modally aligned image and text embeddings for the real data, form per-modality clusters, match clusters across modalities, and average within matched, cross-modally consistent clusters to get prototype embeddings; then decode prototype image embeddings into pixels using a frozen unCLIP decoder (conditioned directly on CLIP image embeddings, with a retrieved real caption as auxiliary conditioning), so the whole pipeline touches no trainable parameters at distillation time.", "method": "Real image and text CLIP embeddings $\\{(z^{img}_n,z^{txt}_n)\\}$ are extracted once, frozen. Each modality is separately clustered into $M$ clusters via mini-batch k-means, where $M$ is the target pair budget. A linear assignment (Hungarian algorithm) matches image clusters to text clusters using a cost that counts shared image-text pairs between them; within each matched cluster pair, embeddings whose partner does not fall in the matched cluster on the other side are discarded, and the remaining embeddings are averaged to form an image prototype and a text prototype. Each image prototype is decoded to pixels by a frozen unCLIP decoder conditioned on the CLIP image embedding, with classifier-free guidance (scale 5.0, 100 sampling steps) and an additional real caption (retrieved as the training-set caption closest to the text prototype) as conditioning; the text prototype embedding itself is stored as the distilled text. No network (CLIP, unCLIP, or any evaluation encoder) is trained during distillation. At evaluation, the frozen CLIP text encoder is paired with a trainable image backbone (ResNet-50 or ViT-Ti/16, distinct from the CLIP image encoder used to build prototypes) fine-tuned on the distilled set.", "evidence": "Flickr30K@100 pairs, ResNet backbone (Table 1): PDS IR@1/5/10 7.9/25.8/37.3, TR@1/5/10 10.2/28.2/39.0, vs LoRS (re-implemented with a CLIP text encoder for fair comparison) IR 6.3/18.6/28.0, TR 9.1/24.3/34.5, vs TESLA-VL IR 4.1/14.7/22.9, TR 6.5/17.8/27.3. @300 pairs, ResNet: IR 14.4/38.1/51.4, TR 18.7/45.0/57.8. COCO@100 pairs, ResNet: IR 2.8/10.0/17.3, TR 4.5/14.0/21.4; @300: IR 5.3/17.2/27.2, TR 7.4/20.7/30.2. Against subset-selection baselines (Table 2, Flickr30K@100, ResNet), the best coreset method (Herding) reaches IR@10 20.1 and TR@10 28.2 versus PDS's 37.3/39.0. Against other learning-free (unimodal, adapted) methods (Table 3, same setting): MGD3 reaches IR@10 17.2, D4M reaches IR@10 9.8, versus PDS's 37.3. Cost (Tables 4-5): PDS synthesizes one image in 9.7s at 4.34GB, versus 1,477.7s at 6.13GB for a pixel-space CLIP- inversion baseline the paper constructs (roughly 150x slower). The key ablations (Table 8: cluster filtering; Table 5: role of image prototypes; Table 6: clustering algorithm) show: discarding embeddings whose modality partner falls outside the matched cluster (filtering) raises IR@10 from 28.1 to 37.3 and TR@10 from 30.6 to 39.0 at 100 pairs; using only the text prototype to retrieve or condition generation (no image-prototype conditioning) drops IR@10 to 26.7-28.7 depending on the ablation variant, an ~9-11 point loss versus using both prototypes; and swapping mini-batch k-means for a Gaussian mixture model gives comparable accuracy (IR@1 8.3 vs 7.9) but 150x more clustering time (Table 7), so k-means is kept for efficiency, not accuracy.", "limitations": "The method depends on a generative model that can be conditioned on the same embedding space as the alignment encoder (unCLIP conditioned on CLIP); the paper notes that stronger alignment encoders such as SigLIP have no equivalent embedding-conditioned decoder available, so PDS cannot currently use them. Prototypes formed by clustering are more influenced by dominant/frequent concepts, so long-tail classes risk under-representation (a limitation the paper argues is shared with subset selection and other distillation baselines, with PDS claimed more robust than those in an appendix study). Both CLIP and unCLIP are trained on natural images, so distilling out-of-domain data (e.g. medical imaging) is stated to fail without fine-tuning those models first, reintroducing a training cost the method otherwise avoids. Evaluation protocol (CLIP text tower, non-CLIP image backbones) differs from the NFNet+BERT protocol used by MTT-VL/LoRS/CovMatch/EDGE, so absolute numbers are not directly comparable across that protocol boundary despite reporting against LoRS and TESLA-VL numbers." }, "notes": "" }, { "id": "accelerating-large-scale-dataset-2026", "type": "paper", "title": "Accelerating Large-Scale Dataset Distillation via Exploration-Exploitation Optimization", "short": "E2D", "arxiv": "2602.15277", "url": null, "date": "2026-02-17", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Muhammad J. Alahmadi", "Peng Gao", "Feiyi Wang", "Dongkuan Xu" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "tier": "catalogued", "links": { "code": "https://github.com/ncsu-dk-lab/E2D", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Speeds up decoupled dataset distillation with a two-phase Exploration-Exploitation Distillation (E2D) schedule: initialization from full real images preserves semantic content, an exploration phase applies uniform updates to locate high-loss regions, and an exploitation phase concentrates further updates there. Reports surpassing prior decoupled methods on ImageNet-1K while running about 18x faster, and improving accuracy on ImageNet-21K at roughly 4.3x the speed.", "abstract": "Dataset distillation compresses the original data into compact synthetic datasets, reducing training time and storage while retaining model performance, enabling deployment under limited resources. Although recent decoupling-based distillation methods enable dataset distillation at large scale, they continue to face an efficiency gap: optimization-based decoupling methods achieve higher accuracy but demand intensive computation, whereas optimization-free decoupling methods are efficient but sacrifice accuracy. To overcome this trade-off, we propose Exploration--Exploitation Distillation (E$^2$D), a simple, practical method that minimizes redundant computation through an efficient pipeline that begins with full-image initialization to preserve semantic integrity and feature diversity. It then uses a two-phase optimization strategy: an exploration phase that performs uniform updates and identifies high-loss regions, and an exploitation phase that focuses updates on these regions to accelerate convergence. We evaluate E$^2$D on large-scale benchmarks, surpassing the state-of-the-art on ImageNet-1K while being $18\\times$ faster, and on ImageNet-21K, our method substantially improves accuracy while remaining $4.3\\times$ faster. These results demonstrate that targeted, redundancy-reducing updates, rather than brute-force optimization, bridge the gap between accuracy and efficiency in large-scale dataset distillation. Code is available at https://github.com/ncsu-dk-lab/E2D.", "notes": "" }, { "id": "efficient-analysis-of-the-2026", "type": "paper", "title": "Efficient Analysis of the Distilled Neural Tangent Kernel", "short": "DNTK", "arxiv": "2602.11320", "url": null, "date": "2026-02-11", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Jamie Mahowald", "Brian Bell", "Alex Ho", "Michael Geyer" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "kernel-closed-form" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Shows that the neural tangent kernel's dependence on per-point Jacobians can be reduced by compressing the input data itself with NTK-tuned dataset distillation, cutting required Jacobian evaluations by 20-100x, and that per-class NTK matrices keep a low effective rank under this reduction. Combines NTK-tuned distillation with existing projection and sketching methods as the distilled neural tangent kernel (DNTK), reporting up to five further orders of magnitude reduction in NTK computation while preserving kernel structure and predictive performance.", "abstract": "Neural tangent kernel (NTK) methods are computationally limited by the need to evaluate large Jacobians across many data points. Existing approaches reduce this cost primarily through projecting and sketching the Jacobian. We show that NTK computation can also be reduced by compressing the data dimension itself using NTK-tuned dataset distillation. We demonstrate that the neural tangent space spanned by the input data can be induced by dataset distillation, yielding a 20-100$\\times$ reduction in required Jacobian calculations. We further show that per-class NTK matrices have low effective rank that is preserved by this reduction. Building on these insights, we propose the distilled neural tangent kernel (DNTK), which combines NTK-tuned dataset distillation with state-of-the-art projection methods to reduce up NTK computational complexity by up to five orders of magnitude while preserving kernel structure and predictive performance.", "notes": "" }, { "id": "shapecond-2026", "type": "paper", "title": "ShapeCond: Fast Shapelet-Guided Dataset Condensation for Time Series Classification", "short": "ShapeCond", "arxiv": "2602.09008", "url": null, "date": "2026-02-09", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Sijia Peng", "Yun Xiong", "Xi Chen", "Yi Xie", "Guanzhi Li", "Yanwei Yu", "Yangyong Zhu", "Zhiqiang Shen" ], "kind": "method", "settings": [ "time-series" ], "families": [ "distribution-matching" ], "tier": "catalogued", "links": { "code": "https://github.com/lunaaa95/ShapeCond", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "ShapeCond condenses time series for classification by extracting shapelets, discriminative local motifs that image-centric condensation methods miss, and using a shapelet-guided optimization strategy to preserve them explicitly, with a synthesis cost that is independent of sequence length. The paper reports large speedups over the prior state of the art (CondTSC), e.g. 29x faster and up to 10,000x faster than naively using shapelets on a 3,000-timestep dataset, while improving downstream accuracy over prior time-series condensation methods.", "abstract": "Time series data supports many domains (e.g., finance and climate science), but its rapid growth strains storage and computation. Dataset condensation can alleviate this by synthesizing a compact training set that preserves key information. Yet most condensation methods are image-centric and often fail on time series because they miss time-series-specific temporal structure, especially local discriminative motifs such as shapelets. In this work, we propose ShapeCond, a novel and efficient condensation framework for time series classification that leverages shapelet-based dataset knowledge via a shapelet-guided optimization strategy. Our shapelet-assisted synthesis cost is independent of sequence length: longer series yield larger speedups in synthesis (e.g., 29$\\times$ faster over prior state-of-the-art method CondTSC for time-series condensation, and up to 10,000$\\times$ over naively using shapelets on the Sleep dataset with 3,000 timesteps). By explicitly preserving critical local patterns, ShapeCond improves downstream accuracy and consistently outperforms all prior state-of-the-art time series dataset condensation methods across extensive experiments. Code is available at https://github.com/lunaaa95/ShapeCond.", "builds_on": [ "timedc-2024" ], "notes": "Family (distribution-matching) is a tentative fit for shapelet-statistic alignment; the abstract does not name the matching type explicitly (see handoff doubts). \"CondTSC\" here is the baseline named timedc-2024 in this atlas." }, { "id": "path-guided-fm-2026", "type": "paper", "title": "Path-Guided Flow Matching for Dataset Distillation", "short": "Path-guided flow matching", "arxiv": "2602.05616", "url": null, "date": "2026-02-05", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Xuhui Li", "Zhengquan Luo", "Xiwei Liu", "Yongqiang Yu", "Zhiqiang Xu" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "autoregressive-flow-synthesis" ], "facets": { "prior": [ "flow", "vae" ], "regime": "training-free", "parameterization": [ "pixels" ], "scale": [ "imagenet-subsets", "imagenet-1k" ] }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "The first flow-matching-based generative distillation framework: conducts class-conditional flow matching in a frozen VAE's latent space so synthesis solves an ODE in a few deterministic steps instead of the long stochastic chain of diffusion, with a continuous path-to-prototype guidance algorithm that keeps trajectories reliably on-path to their assigned prototype while preserving diversity; matches or beats prior diffusion-based distillation with about 7.6x higher efficiency and 78% mode coverage.", "abstract": "Dataset distillation compresses large datasets into compact synthetic sets with comparable performance in training models. Despite recent progress on diffusion-based distillation, this type of method typically depends on heuristic guidance or prototype assignment, which comes with time-consuming sampling and trajectory instability and thus hurts downstream generalization especially under strong control or low IPC. We propose \\emph{Path-Guided Flow Matching (PGFM)}, the first flow matching-based framework for generative distillation, which enables fast deterministic synthesis by solving an ODE in a few steps. PGFM conducts flow matching in the latent space of a frozen VAE to learn class-conditional transport from Gaussian noise to data distribution. Particularly, we develop a continuous path-to-prototype guidance algorithm for ODE-consistent path control, which allows trajectories to reliably land on assigned prototypes while preserving diversity and efficiency. Extensive experiments across high-resolution benchmarks demonstrate that PGFM matches or surpasses prior diffusion-based distillation approaches with fewer steps of sampling while delivering competitive performance with remarkably improved efficiency, e.g., 7.6$\\times$ more efficient than the diffusion-based counterparts with 78\\% mode coverage.", "awesome_sections": [ "Main / Generative Distillation / Flow" ], "builds_on": [ "mgd3-2025", "minimax-2024", "dm-2023" ], "explained": { "before": "Diffusion-guided methods (Minimax, MGD3, DiT-based sampling) generate through a long stochastic denoising chain and steer it with heuristic guidance terms (mode-clustering, representativeness) applied at every one of many timesteps, which the paper identifies as slow and prone to trajectory instability at low IPC or under strong guidance.", "problem": "Heuristic guidance and prototype assignment inside a stochastic diffusion chain make sampling slow (tens of denoising steps, each guided) and unreliable: trajectories can wander off their assigned prototype or collapse toward it too aggressively, hurting downstream generalization, especially when guidance is strong or the per-class sample budget (IPC) is low.", "idea": "Replace the stochastic diffusion chain with class-conditional flow matching, which transports noise to data along (near-)straight, ODE-solvable paths in a few deterministic steps, and add a lightweight control term that steers each trajectory continuously toward its assigned class prototype without fighting the model's own learned velocity field, so guidance is both cheaper (fewer steps) and more reliable (a bounded correction to a deterministic path) than diffusion-time guidance.", "method": "A pretrained GMFlow (Gaussian Mixture Flow matching) model (gmflow_imagenet_k8_ema, trained on ImageNet, frozen -- training-free at the generator level) operates in the latent space of a frozen Stable- Diffusion VAE. Per-class prototypes $\\{\\mu_{y,k}\\}_{k=1}^K$ (K=IPC) are found by K-means++ clustering of the class's real-image latent pool. At sampling time, a prototype control velocity is added to the model's own flow field: $u_{proto}(t,z(t),y,k)=\\lambda(\\mu_{y,k}-\\hat z_1(t))$, where $\\hat z_1(t)$ is the model's current denoised (endpoint) estimate. A trust-region scale $\\alpha(t)$ caps this correction's norm relative to the base velocity's norm so it never dominates, guidance is gated off after $t= t_{end}=0.6$ of the trajectory (early stopping) to avoid distorting fine detail, and initialization is warm-started by interpolating the initial latent toward the assigned prototype by $\\eta_{init}$ (tuned per IPC: 0.09/0.05/0.01 for IPC 10/20/50). Only pixels are stored as the distilled artifact.", "evidence": "ImageNette 256x256, hard-label (Table 1): IPC10, ConvNet-6: 63.0+/-1.1 vs MGD3 58.5+/-2.1; IPC20, ResNet-18: 70.2+/-0.7 vs MGD3 68.1+/-2.1; IPC50, ResNet-AP: 79.2+/-1.3 vs MGD3 79.1+/-1.5 (roughly tied). ImageNet100 (Table 2), IPC10, ConvNet-6: 25.7+/-0.4 vs MGD3 23.4+/-0.9; IPC20, ResNet-18: 32.9+/-0.5 vs MGD3 32.0+/-1.4. ImageNet-1K IPC10 (Table 3): ConvNet-6 14.1 vs MGD3 13.6; ResNet-18 22.8 vs MGD3 21.5. Also compared against DM, DiT, Minimax, and an unguided flow-matching (FM) baseline; results are reported as newly run under the paper's own protocol rather than quoted verbatim. Efficiency (Table 4): PGFM uses 32 NFE vs MGD3's 50 and Minimax's 50, 2.27 TFLOPs vs MGD3's 17.30 and Minimax's 23.73 -- \"10.5x\" efficiency gain over Minimax cited in-table, \"7.6x\" cited in the abstract for the overall diffusion-based comparison. Mode coverage / hit rate (Table 7): FM 10%, MGD3 30%, PGFM (all components) 78%. Component ablation (Table 6, ImageNette IPC10): FM baseline (no guidance) 60.5+/-2.1; + path control 61.6+/-1.2; + early stop 62.1+/-1.3; + trust region 63.0+/-1.1 (best) -- each guidance component adds incrementally. Warm-start sweep (Table 8): eta_init=0 gives 33% hit rate, eta_init=0.09 gives 78%, eta_init=0.15 gives 86% hit rate but lower downstream accuracy (62.0), showing hit rate and accuracy are not monotonically aligned and the paper picks a value short of maximum hit rate.", "limitations": "Stated (Appendix E): performance is bounded by the quality and class-coverage of the underlying pretrained GMFlow model and VAE latent space -- a target-trained-generator dependency in a different form (the flow model must already know the classes it generates), since GMFlow was itself pretrained on ImageNet; the method has several hyperparameters ($\\lambda$, $t_{end}$, $\\rho(t)$, $\\eta_{init}$) whose optimal values vary with dataset scale, resolution and class count; prototype discovery uses simple K-means in latent space, noted as potentially suboptimal for fine-grained or long-tailed classes. Observed: not compared against D3HR, CaO2, IGD, CoDA, DAP, ManifoldGD or any training-free or VLM- conditioned diffusion-guided method, only against DM, DiT, Minimax and MGD3, so its standing against the 2025-2026 diffusion frontier (not just the earlier fine-tuned-prior generation) is untested; the ImageNet-1K result is reported at IPC10 only, no IPC50 ImageNet-1K number given." } }, { "id": "grounding-and-enhancing-informativeness-2026", "type": "paper", "title": "Grounding and Enhancing Informativeness and Utility in Dataset Distillation", "arxiv": "2601.21296", "url": null, "date": "2026-01-29", "venue": { "name": "ICLR 2026", "status": "conference" }, "authors": [ "Shaobo Wang", "Yantai Yang", "Guo Chen", "Peiru Li", "Kaixin Li", "Yufa Zhou", "Zhaorun Chen", "Linfeng Zhang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Defines Informativeness (information within one sample) and Utility (how essential a sample is to the training set) to mathematically characterize optimal dataset distillation, replacing the field's heuristic balancing of efficiency and quality; InfoUtil instantiates both with game-theoretic Shapley-value attribution to extract key per-sample information and gradient-norm-based selection of globally influential samples, reporting a 6.1% accuracy improvement over the prior state of the art on ImageNet-1K with ResNet-18.", "abstract": "Dataset Distillation (DD) seeks to create a compact dataset from a large, real-world dataset. While recent methods often rely on heuristic approaches to balance efficiency and quality, the fundamental relationship between original and synthetic data remains underexplored. This paper revisits knowledge distillation-based dataset distillation within a solid theoretical framework. We introduce the concepts of Informativeness and Utility, capturing crucial information within a sample and essential samples in the training set, respectively. Building on these principles, we define optimal dataset distillation mathematically. We then present InfoUtil, a framework that balances informativeness and utility in synthesizing the distilled dataset. InfoUtil incorporates two key components: (1) game-theoretic informativeness maximization using Shapley Value attribution to extract key information from samples, and (2) principled utility maximization by selecting globally influential samples based on Gradient Norm. These components ensure that the distilled dataset is both informative and utility-optimized. Experiments demonstrate that our method achieves a 6.1\\% performance improvement over the previous state-of-the-art approach on ImageNet-1K dataset using ResNet-18.", "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "towards-realistic-remote-sensing-2026", "type": "paper", "title": "Towards Realistic Remote Sensing Dataset Distillation with Discriminative Prototype-guided Diffusion", "short": "DPD", "arxiv": "2601.15829", "url": null, "date": "2026-01-22", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Yonghao Xu", "Pedram Ghamisi", "Qihao Weng" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "prior": [ "diffusion" ] }, "applications": [ "remote-sensing" ], "tier": "catalogued", "links": { "code": "https://github.com/YonghaoXu/DPD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Introduces dataset distillation to remote sensing image interpretation for the first time with discriminative prototype-guided diffusion (DPD): per-category prototypes in latent space anchor hyperspherical semantic regions that guide the reverse denoising trajectory, and multiple candidates per prototype are ranked by a latent classifier's logit margin so only the most discriminative generated samples form the distilled set. Reports realistic, diverse and discriminative distilled samples across three high-resolution remote sensing scene-classification benchmarks.", "abstract": "Recent years have witnessed the remarkable success of deep learning in remote sensing image interpretation, driven by the availability of large-scale benchmark datasets. However, this reliance on massive training data also brings substantial storage and computational costs. To address this challenge, this study introduces the concept of dataset distillation into the field of remote sensing image interpretation for the first time. Specifically, we propose discriminative prototype-guided diffusion (DPD), a diffusion-based generative distillation framework that condenses a large-scale remote sensing dataset into a compact and representative distilled dataset. To improve the semantic fidelity and diversity of the synthesized samples, we extract representative prototypes for each category in the latent space. We then construct hyperspherical semantic anchors around the prototypes to guide the reverse denoising trajectory. Furthermore, to enhance the discriminative quality of the generated samples, multiple candidates are generated for each prototype and ranked by a latent classifier using a logit-margin criterion, with the most discriminative candidates selected to form the final distilled dataset. Experiments on three high-resolution remote sensing scene classification benchmarks show that the proposed method can distill realistic, diverse, and discriminative samples for downstream model training. Code and pre-trained models are available online (https://github.com/YonghaoXu/DPD).", "awesome_sections": [ "Applications / Remote Sensing" ], "notes": "" }, { "id": "difficulty-guided-sampling-bridging-2026", "type": "paper", "title": "Difficulty-guided Sampling: Bridging the Target Gap between Dataset Distillation and Downstream Tasks", "short": "DGS", "arxiv": "2601.10090", "url": null, "date": "2026-01-15", "venue": { "name": "arXiv 2026", "status": "preprint" }, "authors": [ "Mingzhuo Li", "Guang Li", "Linfeng Ye", "Jiafeng Mao", "Takahiro Ogawa", "Konstantinos N. Plataniotis", "Miki Haseyama" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes", "diffusion-guided-synthesis" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Argues that existing distillation objectives match dataset-level statistics but ignore task-specific information needed for the downstream classifier, creating a target gap; introduces difficulty-guided sampling (DGS), a plug-in post-processing module that resamples an existing method's pool of distilled images to match a target difficulty distribution, and difficulty-aware guidance (DAG), which steers a diffusion generation process by the same difficulty signal. Reports consistent improvements when added to multiple existing distillation settings.", "abstract": "In this paper, we propose difficulty-guided sampling (DGS) to bridge the target gap between the distillation objective and the downstream task, therefore improving the performance of dataset distillation. Deep neural networks achieve remarkable performance but have time and storage-consuming training processes. Dataset distillation is proposed to generate compact, high-quality distilled datasets, enabling effective model training while maintaining downstream performance. Existing approaches typically focus on features extracted from the original dataset, overlooking task-specific information, which leads to a target gap between the distillation objective and the downstream task. We propose leveraging characteristics that benefit the downstream training into data distillation to bridge this gap. Focusing on the downstream task of image classification, we introduce the concept of difficulty and propose DGS as a plug-in post-stage sampling module. Following the specific target difficulty distribution, the final distilled dataset is sampled from image pools generated by existing methods. We also propose difficulty-aware guidance (DAG) to explore the effect of difficulty in the generation process. Extensive experiments across multiple settings demonstrate the effectiveness of the proposed methods. It also highlights the broader potential of difficulty for diverse downstream tasks.", "notes": "" }, { "id": "cd-2-2025", "type": "paper", "title": "CD^2: Constrained Dataset Distillation for Few-Shot Class-Incremental Learning", "short": "CD^2", "arxiv": "2601.08519", "url": null, "date": "2026-01-13", "venue": { "name": "IJCAI 2025", "status": "conference" }, "authors": [ "Kexin Bao", "Daichi Zhang", "Hansong Zhang", "Yong Li", "Yutao Yue", "Shiming Ge" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "meta-learning-bptt" ], "applications": [ "continual-learning" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "CD^2 targets few-shot class-incremental learning by pairing a classifier-guided dataset distillation module, which condenses each new session's few-shot samples into highly compacted exemplars, with a distillation constraint module that regularizes the previously learned class distribution so old sessions' distilled knowledge is not overwritten. The paper reports improvements over other class-incremental methods on three public benchmarks.", "abstract": "Few-shot class-incremental learning (FSCIL) receives significant attention from the public to perform classification continuously with a few training samples, which suffers from the key catastrophic forgetting problem. Existing methods usually employ an external memory to store previous knowledge and treat it with incremental classes equally, which cannot properly preserve previous essential knowledge. To solve this problem and inspired by recent distillation works on knowledge transfer, we propose a framework termed \\textbf{C}onstrained \\textbf{D}ataset \\textbf{D}istillation (\\textbf{CD$^2$}) to facilitate FSCIL, which includes a dataset distillation module (\\textbf{DDM}) and a distillation constraint module~(\\textbf{DCM}). Specifically, the DDM synthesizes highly condensed samples guided by the classifier, forcing the model to learn compacted essential class-related clues from a few incremental samples. The DCM introduces a designed loss to constrain the previously learned class distribution, which can preserve distilled knowledge more sufficiently. Extensive experiments on three public datasets show the superiority of our method against other state-of-the-art competitors.", "awesome_sections": [ "Applications / Continual Learning" ], "notes": "" }, { "id": "towards-data-quality-aware-2026", "type": "paper", "title": "Towards Data Quality-Aware Dataset Distillation in Bioimaging", "arxiv": null, "url": "https://www.scitepress.org/publishedPapers/2026/143417", "date": "2026-01-01", "venue": { "name": "BIOIMAGING 2026", "status": "conference" }, "authors": [ "Bárbara Capelo", "Maria Russo", "André Carreiro", "Hugo Gamboa", "Duarte Folgado" ], "kind": "evaluation-benchmark", "settings": [ "image-classification" ], "families": [], "applications": [ "medical" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "abstract" }, "summary": "A preliminary study of quality-aware dataset distillation for blood cell microscopy images, arguing that fidelity and diversity metrics should be assessed alongside standard downstream performance measures, and examining how different initialization strategies affect the fidelity and diversity of the resulting distilled data.", "abstract": null, "awesome_sections": [ "Applications / Medical" ], "notes": "kind: evaluation-benchmark — the paper is a preliminary quality-assessment study rather than a new method; families left empty since no mechanism is central to its contribution." }, { "id": "attention-hijacking-2026", "type": "paper", "title": "Attention Hijacking: Backdooring Text Dataset Distillation via Semantic Anchors", "short": "Attention Hijacking", "arxiv": null, "url": "https://openreview.net/forum?id=g4BWfkULIo", "date": "2026-01-01", "venue": { "name": "ICML 2026", "status": "conference" }, "authors": [ "Hang Ren" ], "kind": "trustworthy", "settings": [ "text" ], "families": [], "applications": [ "backdoor" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "openreview", "depth": "metadata" }, "summary": "Proposes a backdoor attack on text dataset distillation built on a \"Semantic Anchoring Hypothesis\": the attack reshapes gradients into input embeddings so the synthetic data evolves to turn a trigger word into an adversarial feature, integrated into the bi-level distillation loop so the attack satisfies both the clean-task and backdoor objectives at once.", "abstract": null, "awesome_sections": [ "Applications / Privacy" ], "notes": "OpenReview page blocked by a bot-verification wall; landing-page abstract could not be fetched. Description reconstructed from a search-engine summary of the paper only (status.depth: metadata)." }, { "id": "asymmetric-synthetic-data-update-2026", "type": "paper", "title": "Asymmetric Synthetic Data Update for Domain Incremental Dataset Distillation", "arxiv": null, "url": "https://openreview.net/forum?id=XcsaCHaoJh", "date": "2026-01-01", "venue": { "name": "ICLR 2026", "status": "conference" }, "authors": [ "Minyoung Oh", "Sim" ], "kind": "application", "settings": [ "image-classification" ], "families": [], "applications": [ "continual-learning" ], "tier": "catalogued", "links": { "code": "https://github.com/myoh97/DIDD-ASU", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "openreview", "depth": "abstract" }, "summary": "Introduces Domain Incremental Dataset Distillation, where domains arrive over time and must be continually distilled into a single synthetic set rather than distilled once from a fixed dataset; naively overwriting the synthetic set with each new domain causes catastrophic forgetting. Proposes an Asymmetric Synthetic Data Update strategy that adjusts per-sample update rates to balance stability against plasticity.", "abstract": null, "awesome_sections": [ "Applications / Continual Learning" ], "notes": "Second author name incomplete in available sources (given as \"Sim\" in search results); full name not independently confirmed." }, { "id": "unitsc-2026", "type": "paper", "title": "One Batch Is Enough: A Unified Dataset Condensation Framework for General Time Series Analysis", "short": "UniTSC", "arxiv": null, "url": "https://openreview.net/forum?id=i8FO7f2OYJ", "date": "2026-01", "venue": { "name": "ICML 2026", "status": "conference" }, "authors": [ "Wei Shao", "Ziquan Fang", "Zheqi Lu", "Yongfeng Su", "Yuzhu Wang", "Yunjun Gao" ], "kind": "method", "settings": [ "time-series" ], "families": [ "distribution-matching" ], "tier": "catalogued", "links": { "code": "https://github.com/ZJU-DAILY/UniTSC", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "openreview", "depth": "abstract" }, "summary": "UniTSC argues that prior time-series condensation methods are task-specific (optimized for one downstream task such as forecasting and performing poorly on others such as imputation), and proposes a task-invariant condensation framework that jointly captures temporal, spectral and topological properties of the data so the same condensed set generalizes across multiple time-series analysis tasks.", "abstract": null, "notes": "Non-arXiv (OpenReview access was blocked for direct fetch); title, authors and scope confirmed via independent search excerpts. Family (distribution-matching) is a tentative fit; the exact matching mechanism is not clearly described in the available text." }, { "id": "scg-2026", "type": "paper", "title": "Set-Coupled Guidance: Set-Level Coordination in Diffusion-Based Dataset Distillation", "short": "SCG", "arxiv": null, "url": "https://openreview.net/forum?id=onosLtwoiK", "date": "2026-01", "venue": { "name": "ICML 2026", "status": "conference" }, "authors": [ "Ziang Gan", "Qi Zhu", "Libao Zhang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "tier": "catalogued", "links": { "code": "https://github.com/tade1s/SCG", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "metadata" }, "summary": "Proposes a plug-and-play auxiliary controller that coordinates diffusion sampling across an entire per-class image budget at once ('IPC-at-once') rather than image by image, injecting set-symmetric feedback at each denoising step so the generated set is coordinated as a group instead of independently sampled.", "abstract": null, "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "notes": "OpenReview PDF (both /pdf?id= and /attachment?id= forms) returned only a browser-verification challenge (403/CAPTCHA) on repeated attempts; no arXiv id found by title search; ICML 2026 (July 2026) proceedings are not yet posted to proceedings.mlr.press as of 2026-09. Per the digest task's instruction, full-text reading was skipped rather than proceeding on secondary sources. The GitHub code repo (github.com/tade1s/SCG, built on the MGD3 codebase) is publicly readable and its source (sample_scg.py, argument.py) shows the mechanism operates on a DiT backbone with classifier-free guidance and exposes hyperparameters named 'alpha' (collision-avoidance control strength, a repulsive force between jointly generated samples to maintain diversity) and 'beta' (spectral matching strength, default 0.15, aligning the generated set's PCA-projected feature spectrum to a 'spectral bank' prepared from real data via choices of ecf / rbf_mmd / gaussian_moment / exact_fid distribution objectives) -- consistent with the summary's \"set-symmetric feedback\" description, but no results tables, baselines, or ablation numbers were recoverable from the code alone. Title, authors and method description otherwise sourced from indexed search snippets, not a directly fetched primary abstract. Flag for re-verification once the paper PDF or proceedings volume is accessible." }, { "id": "protovar-2026", "type": "paper", "title": "ProtoVAR: Efficient Dataset Distillation via Prototype-Guided Visual Autoregressive Modeling", "short": "ProtoVAR", "arxiv": null, "url": "https://openreview.net/forum?id=5YvTHX0QS9", "date": "2026-01", "venue": { "name": "ICML 2026", "status": "conference" }, "authors": [ "Mingyu Wang", "Wei Jiang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "autoregressive-flow-synthesis" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "metadata" }, "summary": "Replaces diffusion sampling with a visual autoregressive (VAR) model guided by multi-scale class prototypes, adding a fast filtering stage that keeps only the most informative generated samples; reported to run up to 70x faster than diffusion-based generative distillation while remaining competitive on ImageNet-scale benchmarks.", "abstract": null, "awesome_sections": [ "Main / Generative Distillation / VAR" ], "notes": "OpenReview page returned only a browser-verification challenge (403); title, authors and method description sourced from indexed search snippets and an ML-Anthology-style mirror, not a directly fetched primary abstract. Flag for re-verification once accessible." }, { "id": "parameterization-based-dataset-distillation-2026", "type": "paper", "title": "Parameterization-Based Dataset Distillation of 3D Point Clouds through Learnable Shape Morphing", "short": "3DDP", "arxiv": null, "url": "https://openreview.net/forum?id=Qe7dKZOtWM", "date": "2026-01", "venue": { "name": "ICLR 2026", "status": "conference" }, "authors": [ "Dongwook Kim", "Jae-Young Yim" ], "kind": "method", "settings": [ "other-data" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "factorized-bases" ] }, "tier": "notable", "links": { "code": "https://github.com/donguk071/3DDP", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "openreview", "depth": "abstract" }, "summary": "This method extends parameterization-based dataset distillation to 3D point clouds by storing a small set of coarser-resolution anchor point clouds plus learnable per-sample morphing weights, so that morphing the anchors' shapes generates a more diverse synthetic set than the same memory budget would allow if samples were stored directly; a uniformity-aware matching loss keeps the morphed samples structurally consistent with the originals. The paper reports outperforming existing dataset distillation methods on ModelNet10, ModelNet40, ShapeNet, ScanObjectNN and OmniObject3D.", "abstract": null, "notes": "Non-arXiv (OpenReview access was blocked for direct fetch); title, mechanism and benchmarks confirmed via independent search excerpts of the paper." }, { "id": "ogm-2026", "type": "paper", "title": "Beyond Soft Label: Dataset Distillation via Orthogonal Gradient Matching", "short": "OGM", "arxiv": null, "url": "https://openaccess.thecvf.com/content/CVPR2026/html/Bo_Beyond_Soft_Label_Dataset_Distillation_via_Orthogonal_Gradient_Matching_CVPR_2026_paper.html", "date": "2026-01", "venue": { "name": "CVPR 2026", "status": "conference" }, "authors": [ "Deyu Bo", "Xinchao Wang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "facets": { "labels": "mixed" }, "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "cvf", "depth": "abstract" }, "summary": "Shows theoretically that batch-norm statistic matching in decoupled synthesis mainly aligns the scale of real and synthetic gradients but overlooks their direction, though experiments show direction, not scale, is what drives training; Orthogonal Gradient Matching (OGM) orthogonalizes real and synthetic gradients by fixing all singular values to one and matches their singular vectors with a forward-pass-only least-squares loss, reporting 47.0% top-1 with soft labels and 16.7% with hard labels at IPC=10 on ImageNet-1K, ahead of RDED.", "abstract": null, "builds_on": [ "rded-2024" ], "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "mmdd-2026", "type": "paper", "title": "Efficient Multi-modal Dataset Distillation via Analytic Parameter Matching", "short": "MMDD", "arxiv": null, "url": "https://openreview.net/forum?id=Fxz0aaGSNY", "date": "2026-01", "venue": { "name": "ICML 2026", "status": "conference" }, "authors": [ "Deyu Bo", "Xinchao Wang" ], "kind": "method", "settings": [ "vision-language" ], "families": [ "kernel-closed-form" ], "facets": { "labels": "hard", "prior": [ "pretrained-classifier" ], "regime": "single-level", "parameterization": [ "pixels", "embedding-features" ], "scale": [ "flickr30k-coco" ] }, "tier": "notable", "links": { "code": "https://github.com/bdy9527/MMDD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "openreview", "depth": "abstract" }, "summary": "MMDD replaces the bi-trajectory distillation used by prior multimodal dataset distillation methods (aligning expert and student trajectories per modality, which needs many stored checkpoints and double backpropagation) with analytic parameter matching, directly matching the closed-form analytic parameters of the modal projectors, which the paper connects theoretically to matrix whitening. It caches only two matrices instead of trajectory checkpoints and computes the match in a single forward pass, reporting up to 65x storage reduction and 9.6x distillation speedup while scaling to 1000 synthetic pairs on Flickr30k and MS-COCO.", "abstract": null, "builds_on": [ "lors-2024" ], "notes": "Full-text NOT obtained despite repeated attempts this session: the OpenReview forum page and both the id-based (/pdf?id=...) and hash-based direct PDF URLs all return only OpenReview's browser verification/CAPTCHA challenge (confirmed via WebFetch and a direct curl with a browser user-agent, which returned HTTP 403 with the same challenge HTML); the GitHub repo (bdy9527/MMDD) render did not expose README content through the fetch tool either. status.depth is therefore left at \"abstract\" per the digest guide (do not set full-text without reading the text). Corrected the OpenReview forum id in `url` from the draft's Yh4dMR5mJ0 (does not resolve to this paper) to Fxz0aaGSNY, which search results consistently associate with this exact title. Added a second author, Xinchao Wang, found via a secondary listing (a \"Lacuna\" paper-tracking page mirroring the OpenReview abstract) alongside Deyu Bo; still not verified against the primary OpenReview page itself, so the author list should be re-checked when access is available. That same secondary source gives additional unverified detail worth flagging for a future full-text pass: the 65x storage figure is stated as 32.6GB (LoRS) to 0.5GB (APM), and the 9.6x speedup as 11.5s/iteration (LoRS) to 1.2s/iteration (APM) - the 11.5s LoRS figure matches repblend-2025's own independently-reported LoRS per-iteration cost (Table 4 in that paper), which is a point of corroboration though still not a primary-source confirmation for MMDD itself. The same source states a Flickr30k@100-pair IR@1 of 12.8% for APM versus RepBlend 11.5% and LoRS 8.3%, and describes a stated limitation that the method assumes strict 1:1 image-caption correspondence, requiring a workaround for multi-caption datasets like COCO. None of these secondary numbers should be treated as confirmed; they are recorded only to help a future re-attempt know what to verify." }, { "id": "ddshift-2026", "type": "paper", "title": "Mitigating the Distribution Shift of Diffusion-based Dataset Distillation", "short": "Distribution shift in diffusion DD", "arxiv": null, "url": "https://openaccess.thecvf.com/content/CVPR2026/html/Xu_Mitigating_The_Distribution_Shift_of_Diffusion-based_Dataset_Distillation_CVPR_2026_paper.html", "date": "2026-01", "venue": { "name": "CVPR 2026", "status": "conference" }, "authors": [ "Yue Xu", "Chenyu Hu", "Pengyu An", "Yong-Lu Li" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "prior": [ "diffusion" ], "regime": "generator-fine-tuning" }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "cvf", "depth": "abstract" }, "summary": "Identifies two distribution shifts in diffusion-based distillation: an optimal synthetic distribution should simplify, not replicate, the real data distribution given the synthetic set's small capacity, and the sampling process itself introduces a further deviation from the learned distribution; fixes both with an L1 sparsity regularizer during diffusion training and, at sampling time, a synchronous (non-sequential) denoising of the whole synthetic dataset with distribution regularizers.", "abstract": null, "awesome_sections": [ "Main / Generative Distillation / Diffusion" ] }, { "id": "bps-2026", "type": "paper", "title": "Balanced Dataset Distillation via Modeling Multiple Visual Pattern Distribution", "short": "BPS", "arxiv": null, "url": "https://openaccess.thecvf.com/content/CVPR2026/html/Shi_Balanced_Dataset_Distillation_via_Modeling_Multiple_Visual_Pattern_Distribution_CVPR_2026_paper.html", "date": "2026-01", "venue": { "name": "CVPR 2026", "status": "conference" }, "authors": [ "Guanghui Shi", "Xuefeng Liang", "Qixiang Wen" ], "kind": "baseline", "settings": [ "image-classification" ], "families": [ "coreset-selection" ], "tier": "catalogued", "links": { "code": "https://github.com/BeCarefulOfYournaoke/BPS", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "cvf", "depth": "abstract" }, "summary": "Models each class as multiple visual-pattern clusters via a hierarchical semantic structure, instead of assuming one cluster per class, and selects a balanced coreset of complementary class-general (center) and marginal (boundary) real samples from every cluster; proves the selected coreset aligns with the original dataset in information coverage and reports state-of-the-art results across four benchmarks with model-agnostic, one-shot-for-all-IPCs selection.", "abstract": null, "awesome_sections": [ "Main / Decoupled Distillation" ], "notes": "Reclassified from decoupled-synthesis (draft's Awesome-list section) to coreset-selection: the method selects real samples via hierarchical clustering, it does not synthesize or invert against a teacher." }, { "id": "amd-2026", "type": "paper", "title": "Asynchronous Matching with Dynamic Sampling for Multimodal Dataset Distillation", "short": "AMD", "arxiv": null, "url": "https://openreview.net/forum?id=7SgSMKM2KF", "date": "2026-01", "venue": { "name": "ICLR 2026", "status": "conference" }, "authors": [ "Ding Qi", "Jian Li", "Shuguang Dou", "Zifan Song", "Junyao Gao", "Yabiao Wang", "Chengjie Wang", "Cairong Zhao" ], "kind": "method", "settings": [ "vision-language" ], "families": [ "trajectory-matching" ], "facets": { "labels": "hard", "prior": [ "trained-experts" ], "regime": "bilevel", "parameterization": [ "pixels", "embedding-features" ], "scale": [ "flickr30k-coco" ] }, "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-14", "source": "openreview", "depth": "abstract" }, "summary": "Proposes AMD for image-text dataset distillation, performing asynchronous trajectory matching that decouples the starting points of image and text expert trajectories to accommodate their different learning speeds, and replacing random prototype initialization with a Semantics-Aware Prototype Mining module that clusters the feature space into representative prototypes; reported Recall@1/5/10 gains are 4.5, 9.6 and 10.9 points on Flickr30K at 200 distilled pairs.", "abstract": null, "builds_on": [ "mtt-vl-2024", "lors-2024" ], "awesome_sections": [ "Main / Multimodal Distillation" ], "notes": "Full-text NOT obtained despite repeated attempts this session: the OpenReview forum page, the id-based PDF URL, and a hash-based direct PDF URL found via search (openreview.net/pdf/0cdc6c37e7048387fe506c50c23225e7e3ca5f29.pdf) all return only OpenReview's browser verification/CAPTCHA challenge (confirmed via WebFetch and a direct curl with a browser user-agent, HTTP 403 with the same challenge page); a jina.ai reader-proxy fetch of the same PDF URL was also blocked by the same challenge. status.depth is therefore left at \"abstract\" per the digest guide. Two independent secondary sources beyond the ICLR virtual site were checked this session and agree with each other and with the previously recorded numbers: ML Anthology's listing (full author list confirmed matching the draft) and a third-party paper-notes page (en.papernotes.org/ICLR2026/multimodal_vlm/...), which additionally describes the mechanism as decoupling trajectory *starting points* $(t_v,t_l)$ rather than forcing image/text trajectories to start at the same step, an MMD-based dynamic-sampling rule that sets each modality's per-step sampling range from the ratio of its parameter-update magnitudes between epochs, and reports a Flickr30k 200-pair ablation table (baseline/LoRS IR@1/5/10 8.6/25.3/36.6; +AMD-matching-alone 12.1/33.9/46.7; +prototype-mining-alone 9.1/26.4/38.5; +both 13.1/34.9/47.5) plus a claim of \"zero additional overhead\" and a separate CLIP-backbone result of 47.9 IR@1 at a 10% synthetic budget (\"over 96%\" of a 49.8 full-dataset ceiling). None of this is a primary-source confirmation - it is a third party's summary of the paper, not the paper itself - so these numbers are recorded for a future re-verification pass, not treated as confirmed. Flagged for a re-attempt from an environment where OpenReview is not blocked, or if the authors release code/an arXiv version." }, { "id": "secure-and-explainable-fraud-2025", "type": "paper", "title": "Secure and Explainable Fraud Detection in Finance via Hierarchical Multi-source Dataset Distillation", "arxiv": "2512.21866", "url": null, "date": "2025-12-26", "venue": { "name": "ICAIFW 2025", "status": "conference" }, "authors": [ "Yiming Qian", "Thorsten Neumann", "Xueyining Huang", "David Hardoon", "Fei Gao", "Yong Liu", "Siow Mong Rick Goh" ], "kind": "application", "settings": [ "other-data" ], "families": [], "applications": [ "finance", "privacy" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Converts a trained random forest into transparent axis-aligned rule regions (leaf hyperrectangles) and generates synthetic transactions by uniformly sampling within each region, producing a compact, auditable surrogate dataset with rule-based explanations and calibrated uncertainty for multi-institution fraud detection. On the IEEE-CIS fraud dataset, reports 85-93% data volume reduction with competitive precision/micro-F1 and only a modest AUC drop, chance-level membership-inference attack success, and improved cross-cluster metrics when institutions share the distilled data.", "abstract": "We propose an explainable, privacy-preserving dataset distillation framework for collaborative financial fraud detection. A trained random forest is converted into transparent, axis-aligned rule regions (leaf hyperrectangles), and synthetic transactions are generated by uniformly sampling within each region. This produces a compact, auditable surrogate dataset that preserves local feature interactions without exposing sensitive original records. The rule regions also support explainability: aggregated rule statistics (for example, support and lift) describe global patterns, while assigning each case to its generating region gives concise human-readable rationales and calibrated uncertainty based on tree-vote disagreement. On the IEEE-CIS fraud dataset (590k transactions across three institution-like clusters), distilled datasets reduce data volume by 85% to 93% (often under 15% of the original) while maintaining competitive precision and micro-F1, with only a modest AUC drop. Sharing and augmenting with synthesized data across institutions improves cross-cluster precision, recall, and AUC. Real vs. synthesized structure remains highly similar (over 93% by nearest-neighbor cosine analysis). Membership-inference attacks perform at chance level (about 0.50) when distinguishing training from hold-out records, suggesting low memorization risk. Removing high-uncertainty synthetic points using disagreement scores further boosts AUC (up to 0.687) and improves calibration. Sensitivity tests show weak dependence on the distillation ratio (AUC about 0.641 to 0.645 from 6% to 60%). Overall, tree-region distillation enables trustworthy, deployable fraud analytics with interpretable global rules, per-case rationales with quantified uncertainty, and strong privacy properties suitable for multi-institution settings and regulatory audit.", "awesome_sections": [ "Applications / Finance" ], "notes": "families left empty: the mechanism (sampling within decision-tree leaf hyperrectangles) is not one of the atlas condensation families." }, { "id": "hald-2026", "type": "paper", "title": "Hard Labels In! Rethinking the Role of Hard Labels in Mitigating Local Semantic Drift", "short": "HALD", "arxiv": "2512.15647", "url": null, "date": "2025-12-17", "venue": { "name": "ICML 2026", "status": "conference" }, "authors": [ "Jiacheng Cui", "Bingkui Tong", "Xinyue Bi", "Xiaohan Zhao", "Jiacheng Liu", "Zhiqiang Shen" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "label-distillation", "decoupled-synthesis" ], "facets": { "labels": "mixed", "prior": [ "pretrained-classifier" ], "regime": "decoupled", "parameterization": [ "pixels" ], "scale": [ "imagenet-1k", "tiny-imagenet" ] }, "tier": "notable", "links": { "code": "https://github.com/Jiacheng8/HALD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "SRe2L stores one soft label per crop, and LPLD reduces label-storage cost by capping the number of stored soft labels per class (SLC) and reusing them across training epochs. GIFT mixes hard information into soft targets directly; the paper's own Joint Objective baseline sums a soft and a hard loss with a fixed weight.", "problem": "Storing per-crop soft labels is expensive (ImageNet-1K distilled images: 750 MB vs. soft labels: 28.33 GB, i.e. up to 38x larger, App. F.2/Table 21). Reducing crops per image (LPLD's approach) saves storage but the paper shows it introduces Local-View Semantic Drift (LVSD): with only a few stored crop-level soft labels, ambiguous local crops can have teacher predictions that point toward the wrong class, and Theorem 3.5/3.6 formally lower-bound the resulting train/ideal-objective gap and excess generalization loss at $\\Omega(1/s)$ in the number of stored crops $s$, vanishing only as $s\\to\\infty$.", "idea": "Hard labels are content-invariant (a crop's ground-truth class does not depend on which part of the image it shows), so briefly training on ground-truth-anchored, heavily label-smoothed CutMix targets between two soft-label training phases acts as a variance-reducing corrective signal that recalibrates the model before returning to fine-grained soft supervision, without needing to store any additional labels.", "method": "A three-phase Soft-Hard-Soft schedule applied to an already-distilled image set (default: images generated by FADRM; also tested on SRe2L/LPLD/RDED images). Phase A trains on a fixed pool of $s{=}\\text{SLI}\\times\\text{IPC}$ pre-generated teacher soft labels (crop-level, from the frozen teacher) for $T_A=\\lfloor n_{\\text{soft}}/2\\rfloor$ epochs. Phase B (de-LVSD) trains for $n_{\\text{hard}}$ epochs on freshly-resampled crops and CutMix pairs labeled with heavily label-smoothed, ground-truth-anchored targets $t_{\\lambda,\\alpha}(y,y')$ (not strict one-hot) — zero additional storage since these targets are computed on the fly from the class index, not the teacher. Phase C resumes Phase A's fixed soft-label pool for the remaining $T_C$ epochs. No new teacher, generator, or image synthesis is introduced; the mechanism is purely a training-schedule change over an existing soft-label budget. Theoretical support: Theorem 3.7 shows soft- and hard-label gradients become increasingly cosine-aligned as training converges (validated in Fig. 3), and Corollary 3.8 argues this alignment lets hard-label calibration act as if it increased the effective number of soft-label samples.", "evidence": "ImageNet-1K, ResNet-18, using FADRM-generated images, relabel+KD from a fixed soft-label pool plus HALD's label-smoothed hard-calibration phase (Table 2): at SLI=5 (SLC=250, 475 MB), IPC=50, HALD 49.5% vs its own re-run Soft-Only baselines FADRM 45.5%, LPLD 39.4%, SRe2L 39.5%, RDED 34.9%; at SLI=10 (SLC=500, 950 MB), IPC=50, HALD 53.7% vs FADRM 52.7%, LPLD 48.6% (marked as the reported number). IPC=10, SLI=5 (SLC=50, 95 MB): HALD 30.3% vs LPLD 14.5%, FADRM 16.1%. Storage-matched headline (Table 5, IPC=50): at a fixed 285 MB budget, HALD 42.7% vs LPLD 33.7% (+9.0 points); at 95 MB, HALD 36.9% vs LPLD 14.3% (+22.6); this is the abstract's headline \"42.7% with 285 MB (100x storage reduction vs. per-crop FKD's 28.33 GB), +9.0 over LPLD.\" No image synthesis time is reported since HALD reuses an existing generation method's images; only label-side storage is measured. Ablation isolating the mechanism (Table 3): a Joint Objective (single loss = soft + $ \\lambda$*hard) baseline peaks at $\\lambda{=}0$ and degrades as $\\lambda$ increases, i.e. naively mixing hard and soft losses in one objective *hurts*, while HALD's staged soft-hard-soft schedule with the identical label budget outperforms both Soft-Only and Joint Objective — isolating the staged schedule itself, not merely \"using hard labels,\" as the source of the gain. Table 15 (Appendix) further shows teacher-prediction agreement on unseen data improves after the final soft-refinement phase compared to omitting it, isolating that phase's contribution. No result is reported for a schedule that skips Phase A or Phase C (pure hard-label training throughout).", "limitations": "HALD is a training-schedule/label-regime method layered on top of an existing image generator (FADRM by default); it introduces no new synthesis and its accuracy gain depends on the quality of whichever generation method supplies the underlying images (its own Table 2 shows the gain over Soft-Only shrinks or reorders across SRe2L/RDED/LPLD/FADRM images). The hard-label phase uses heavily label-smoothed CutMix targets rather than strict one-hot labels, so it is not a test of pure hard-label training, and no ablation isolates $\\alpha{=}0$ (true one-hot) hard supervision. The soft/hard/soft epoch split depends on estimating $n_{\\text{soft}}$, the convergence budget for the soft-only phase, which the paper sets heuristically rather than deriving in closed form. Results are reported only on Tiny-ImageNet and ImageNet-1K classification; no result on CIFAR or on datasets without a decoupled-synthesis-style crop/relabel evaluation pipeline is given (though Table 6 extends to CIFAR-100/ImageWoof for the ablation set). The paper's own conclusion flags a risk of reinforcing bias present in hard labels and over-calibrating to a specific dataset." }, "summary": "Shows theoretically and empirically that limiting soft-label crops per image to save storage causes local semantic drift -- visually ambiguous crops pull soft supervision away from the image's true label -- and that hybridizing hard labels back in as a content-invariant semantic anchor corrects it; HALD uses hard labels as an intermediate corrective signal alongside sparse soft labels, reaching 42.7% top-1 on ImageNet-1K with only 285MB of soft-label storage (a 100x reduction) and a 9.0-point gain over LPLD.", "abstract": "Soft labels from teacher models are a de facto practice for knowledge transfer and large-scale dataset distillation (e.g., SRe2L, LPLD). However, when we limit the number of crops per image to reduce the substantial cost of storing precomputed soft labels, these methods suffer severely from local semantic drift: visually ambiguous crops can cause soft supervision to deviate from the image-level ground-truth semantics, leading to persistent errors and a train-test distribution mismatch. We revisit the overlooked role of hard labels and show that, when properly integrated, they can act as a content-invariant semantic anchor that calibrates such drift. We theoretically analyze the emergence of drift under sparse soft-label supervision and demonstrate that hybridizing hard and soft labels restores alignment between visual content and semantic supervision. Building on this insight, we propose a new training paradigm, Hard Label for Alleviating Local Semantic Drift (HALD), which uses hard labels as intermediate corrective signals while preserving the fine-grained benefits of soft labels. Extensive experiments on dataset distillation and large-scale classification benchmarks show consistent generalization improvements. On ImageNet-1K, our method achieves 42.7% accuracy with only 285M soft-label storage (reduces by 100X), outperforming prior state-of-the-art LPLD 9.0%.", "builds_on": [ "sre2l-2023", "lpld-2024", "rded-2024", "fadrm-2025" ], "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "distill-video-datasets-into-2025", "type": "paper", "title": "Distill Video Datasets into Images", "short": "SFVD", "arxiv": "2512.14621", "url": null, "date": "2025-12-16", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Zhenghao Zhao", "Haoxuan Wang", "Kai Wang", "Yuzhang Shang", "Yuan Hong", "Yan Yan" ], "kind": "method", "settings": [ "video" ], "families": [], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Identifies that the temporal dimension explodes the parameter count in video dataset distillation and proposes SFVD, which distills each class into a small set of highly informative single frames and reconstructs clip-length sequences from them via differentiable interpolation, restricting optimization to the frames themselves and mixing in sampled real clips through a channel-reshaping layer during matching; reports up to a 5.3-point improvement over prior methods on MiniUCF.", "abstract": "Dataset distillation aims to synthesize compact yet informative datasets that allow models trained on them to achieve performance comparable to training on the full dataset. While this approach has shown promising results for image data, extending dataset distillation methods to video data has proven challenging and often leads to suboptimal performance. In this work, we first identify the core challenge in video set distillation as the substantial increase in learnable parameters introduced by the temporal dimension of video, which complicates optimization and hinders convergence. To address this issue, we observe that a single frame is often sufficient to capture the discriminative semantics of a video. Leveraging this insight, we propose Single-Frame Video set Distillation (SFVD), a framework that distills videos into highly informative frames for each class. Using differentiable interpolation, these frames are transformed into video sequences and matched with the original dataset, while updates are restricted to the frames themselves for improved optimization efficiency. To further incorporate temporal information, the distilled frames are combined with sampled real videos from real videos during the matching process through a channel reshaping layer. Extensive experiments on multiple benchmarks demonstrate that SFVD substantially outperforms prior methods, achieving improvements of up to 5.3% on MiniUCF, thereby offering a more effective solution.", "awesome_sections": [ "Applications / Video" ], "notes": "Abstract does not name the underlying matching mechanism used to match interpolated clips against the dataset; family left empty." }, { "id": "dire-2026", "type": "paper", "title": "DiRe: Diversity-promoting Regularization for Dataset Condensation", "short": "DiRe", "arxiv": "2512.13083", "url": null, "date": "2025-12-15", "venue": { "name": "WACV 2026", "status": "conference" }, "authors": [ "Saumyaranjan Mohanty", "Aravind Reddy", "Konda Reddy Mopuri" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/DIL-IITH/DiRe", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Proposes DiRe, a diversity regularizer combining cosine similarity and Euclidean distance terms that plugs into existing condensation methods off the shelf to reduce redundancy among synthesized samples; reports consistent generalization and diversity-metric improvements when added to state-of-the-art condensation methods from CIFAR-10 to ImageNet-1K.", "abstract": "In Dataset Condensation, the goal is to synthesize a small dataset that replicates the training utility of a large original dataset. Existing condensation methods synthesize datasets with significant redundancy, so there is a dire need to reduce redundancy and improve the diversity of the synthesized datasets. To tackle this, we propose an intuitive Diversity Regularizer (DiRe) composed of cosine similarity and Euclidean distance, which can be applied off-the-shelf to various state-of-the-art condensation methods. Through extensive experiments, we demonstrate that the addition of our regularizer improves state-of-the-art condensation methods on various benchmark datasets from CIFAR-10 to ImageNet-1K with respect to generalization and diversity metrics.", "awesome_sections": [ "Main / Decoupled Distillation" ], "notes": "Reclassified from decoupled-synthesis to optimization-recipes: the abstract presents DiRe as a general plug-in regularizer for 'various state-of-the-art condensation methods', not specific to the decoupled/teacher-driven pipeline." }, { "id": "eeg-dlite-2026", "type": "paper", "title": "EEG-DLite: Dataset Distillation for Efficient Large EEG Model Training", "short": "EEG-DLite", "arxiv": "2512.12210", "url": null, "date": "2025-12-13", "venue": { "name": "AAAI 2026", "status": "conference" }, "authors": [ "Yuting Tang", "Weibang Jiang", "Shanglin Li", "Yong Li", "Chenyu Liu", "Xinliang Zhou", "Yi Ding", "Cuntai Guan" ], "kind": "method", "settings": [ "other-data" ], "families": [ "coreset-selection" ], "facets": { "regime": "selection" }, "tier": "catalogued", "links": { "code": "https://github.com/t170815518/EEG-DLite", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Introduces EEG-DLite, a data-selection framework for EEG foundation-model pre-training that encodes EEG segments into compact latent representations with a self-supervised autoencoder and then filters outliers and redundant samples in that latent space; reports that training on only 5% of a 2,500-hour EEG corpus curated this way matches or exceeds training on the full dataset across several downstream tasks.", "abstract": "Large-scale EEG foundation models have shown strong generalization across a range of downstream tasks, but their training remains resource-intensive due to the volume and variable quality of EEG data. In this work, we introduce EEG-DLite, a data distillation framework that enables more efficient pre-training by selectively removing noisy and redundant samples from large EEG datasets. EEG-DLite begins by encoding EEG segments into compact latent representations using a self-supervised autoencoder, allowing sample selection to be performed efficiently and with reduced sensitivity to noise. Based on these representations, EEG-DLite filters out outliers and minimizes redundancy, resulting in a smaller yet informative subset that retains the diversity essential for effective foundation model training. Through extensive experiments, we demonstrate that training on only 5 percent of a 2,500-hour dataset curated with EEG-DLite yields performance comparable to, and in some cases better than, training on the full dataset across multiple downstream tasks. To our knowledge, this is the first systematic study of pre-training data distillation in the context of EEG foundation models. EEG-DLite provides a scalable and practical path toward more effective and efficient physiological foundation modeling. The code is available at https://github.com/t170815518/EEG-DLite.", "awesome_sections": [ "Applications / EEG" ], "notes": "" }, { "id": "geodm-2026", "type": "paper", "title": "GeoDM: Geometry-aware Distribution Matching for Dataset Distillation", "short": "GeoDM", "arxiv": "2512.08317", "url": null, "date": "2025-12-09", "venue": { "name": "ICML 2026", "status": "conference" }, "authors": [ "Xuhui Li", "Zhengquan Luo", "Zihui Cui", "Zhiqiang Xu" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Extends distribution matching to a product of Euclidean, hyperbolic and spherical manifolds with learnable curvature per geometry, so the synthetic-data manifold can capture the flat, hierarchical and cyclical structure of the real data rather than only Euclidean statistics, adding an optimal-transport loss; reports a smaller generalization-error bound than Euclidean distribution matching and gains over state-of-the-art distillation methods on standard benchmarks.", "abstract": "Dataset distillation aims to synthesize a compact subset of the original data, enabling models trained on it to achieve performance comparable to those trained on the original large dataset. Existing distribution-matching methods are confined to Euclidean spaces, making them only capture linear structures and overlook the intrinsic geometry of real data, e.g., curvature. However, high-dimensional data often lie on low-dimensional manifolds, suggesting that dataset distillation should have the distilled data manifold aligned with the original data manifold. In this work, we propose a geometry-aware distribution-matching framework, called \\textbf{GeoDM}, which operates in the Cartesian product of Euclidean, hyperbolic, and spherical manifolds, with flat, hierarchical, and cyclical structures all captured by a unified representation. To adapt to the underlying data geometry, we introduce learnable curvature and weight parameters for three kinds of geometries. At the same time, we design an optimal transport loss to enhance the distribution fidelity. Our theoretical analysis shows that the geometry-aware distribution matching in a product space yields a smaller generalization error bound than the Euclidean counterparts. Extensive experiments conducted on standard benchmarks demonstrate that our algorithm outperforms state-of-the-art data distillation methods and remains effective across various distribution-matching strategies for the single geometries.", "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ], "builds_on": [ "dm-2023" ] }, { "id": "utility-boundary-2026", "type": "paper", "title": "Utility Boundary of Dataset Distillation: Scaling and Configuration-Coverage Laws", "short": "Utility boundary laws", "arxiv": "2512.05817", "url": null, "date": "2025-12-05", "venue": { "name": "ICML 2026", "status": "conference" }, "authors": [ "Zhengquan Luo", "Zhiqiang Xu" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Proposes a unified 'configuration-dynamics-error' framework that reformulates gradient-, distribution- and trajectory-matching dataset distillation as interchangeable surrogates reducing the same generalization error, deriving a scaling law bounding error against distilled sample size (explaining observed performance saturation) and a coverage law showing the required sample size grows linearly with training-configuration diversity, both with matching upper and lower bounds confirmed experimentally.", "abstract": null, "awesome_sections": [ "Main / Better Understanding" ] }, { "id": "text-dd-report-2025", "type": "paper", "title": "Technical Report on Text Dataset Distillation", "short": "Text dataset distillation report", "arxiv": "2512.03967", "date": "2025-12-03", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Keith Ando Ogawa", "Bruno Lopes Yamamoto", "Lucas Lauton de Alcantara", "Victor Zacarias", "Edson Bollis", "Lucas Pellicer", "Rosimeire Pereira Costa", "Anna Helena Reali Costa", "Artur Jordao" ], "kind": "survey", "settings": [ "text" ], "families": [], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-14", "source": "arxiv-api", "depth": "abstract" }, "summary": "A review of dataset distillation for text, tracing its path from adaptations of vision methods on embeddings, through transformer-based methods and the generation of discrete synthetic text, to distillation for decoder-only models above one billion parameters, and naming benchmark standardization, the discreteness of text, and complex tasks as the main open problems.", "abstract": "In the vision domain, dataset distillation arises as a technique to condense a large dataset into a smaller synthetic one that exhibits a similar result in the training process. While image data presents an extensive literature of distillation methods, text dataset distillation has fewer works in comparison. Text dataset distillation initially grew as an adaptation of efforts from the vision universe, as the particularities of the modality became clear obstacles, it rose into a separate branch of research. Several milestones mark the development of this area, such as the introduction of methods that use transformer models, the generation of discrete synthetic text, and the scaling to decoder-only models with over 1B parameters. Despite major advances in modern approaches, the field remains in a maturing phase, with room for improvement on benchmarking standardization, approaches to overcome the discrete nature of text, handling complex tasks, and providing explicit examples of real-world applications. In this report, we review past and recent advances in dataset distillation for text, highlighting different distillation strategies, key contributions, and general challenges." }, { "id": "coda-2026", "type": "paper", "title": "CoDA: From Text-to-Image Diffusion Models to Training-Free Dataset Distillation", "short": "CoDA", "arxiv": "2512.03844", "url": null, "date": "2025-12-03", "venue": { "name": "ICLR 2026", "status": "conference" }, "authors": [ "Letian Zhou", "Songhua Liu", "Xinchao Wang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "prior": [ "text-to-image-diffusion" ], "regime": "training-free", "parameterization": [ "pixels" ], "scale": [ "imagenet-subsets", "imagenet-1k", "large-scale-other" ] }, "tier": "notable", "links": { "code": "https://github.com/zzzlt422/CoDA", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Uses an off-the-shelf text-to-image diffusion model, with no target-dataset fine-tuning, by first discovering the target dataset's 'intrinsic core distribution' with a density-based mechanism and then steering sampling to align with it; reports 60.4% top-1 at IPC 50 on ImageNet-1K, matching or beating methods that require a dataset-specific diffusion model.", "abstract": "Prevailing Dataset Distillation (DD) methods leveraging generative models confront two fundamental limitations. First, despite pioneering the use of diffusion models in DD and delivering impressive performance, the vast majority of approaches paradoxically require a diffusion model pre-trained on the full target dataset, undermining the very purpose of DD and incurring prohibitive training costs. Second, although some methods turn to general text-to-image models without relying on such target-specific training, they suffer from a significant distributional mismatch, as the web-scale priors encapsulated in these foundation models fail to faithfully capture the target-specific semantics, leading to suboptimal performance. To tackle these challenges, we propose Core Distribution Alignment (CoDA), a framework that enables effective DD using only an off-the-shelf text-to-image model. Our key idea is to first identify the \"intrinsic core distribution\" of the target dataset using a robust density-based discovery mechanism. We then steer the generative process to align the generated samples with this core distribution. By doing so, CoDA effectively bridges the gap between general-purpose generative priors and target semantics, yielding highly representative distilled datasets. Extensive experiments suggest that, without relying on a generative model specifically trained on the target dataset, CoDA achieves performance on par with or even superior to previous methods with such reliance across all benchmarks, including ImageNet-1K and its subsets. Notably, it establishes a new state-of-the-art accuracy of 60.4% at the 50-images-per-class (IPC) setup on ImageNet-1K. Our code is available on the project webpage: https://github.com/zzzlt422/CoDA", "builds_on": [ "d4m-2024", "mgd3-2025", "minimax-2024" ], "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "explained": { "before": "Minimax, LD3M, IGD and MGD3 fine-tune or otherwise rely on a diffusion model (often a DiT) that was itself pretrained on the full target dataset, e.g. Minimax's DiT backbone needed roughly 3,000 TPU-days of ImageNet pretraining before distillation could start. D4M avoids that by using a general text-to-image latent diffusion model but only initializes sampling from noised K-means prototypes and then lets the model denoise unguided.", "problem": "Two failures, argued from Figure 1: (1) target-trained generators reintroduce the very cost DD is meant to avoid (\"putting the cart before the horse\"); (2) a truly off-the-shelf text-to-image model (SDXL) suffers distributional mismatch -- its web-scale prior samples cluster in a narrow region that does not cover the target class manifold (52% vs 60% for a target-pretrained LDM on ImageNette IPC10) -- and D4M's initialize-then-drift approach lets the general prior override the informative prototype during unguided denoising.", "idea": "Identify the dataset's \"intrinsic core distribution\" -- its high-density, most representative real samples -- with a density-based (not K-means) discovery pipeline in VAE latent space, then continuously steer an off-the-shelf diffusion model's denoising trajectory toward one assigned representative sample per output image for the entire sampling run, rather than only at initialization.", "method": "Distribution Discovery: VAE-encode each class's images (no target-trained feature extractor), apply UMAP dimensionality reduction (tuned to increase local density contrast) then HDBSCAN (robust to non-convex clusters and outliers, unlike K-means); take each cluster's highest-membership point as a representative sample. Since HDBSCAN's cluster count rarely equals IPC, a 3-strategy post-processing scheme (SplitCluster via recursive HDBSCAN+K-means seeding, K-means on the leftover outlier pool, and ForcedSplit with a relaxed min_cluster_size) forces the representative set $S_r$ to exactly IPC samples, deliberately keeping outliers usable rather than discarding them (unlike LOF-based purification, e.g. in VLCP/D4M). Distribution Alignment: at every denoising step, from the CFG text-conditional prediction $\\epsilon_\\theta(z_t,t,c)$ the clean-latent estimate $\\hat z_0(z_t)$ is computed in closed form, a guidance vector $g(z_t)=s_j-\\hat z_0(z_t)$ pulls it toward the $j$-th representative sample $s_j\\in S_r$ (one fixed real sample assigned per generated image for the whole trajectory), scaled by strength $\\gamma$ and algebraically converted into an equivalent additive correction $\\Delta\\epsilon_\\theta$ in noise space so it composes with standard classifier-free guidance; the last \"Prior Injection Steps\" (PIS) revert to plain CFG so the model's own prior can finish photorealistic detail. Base generator: SDXL, DPM++ Karras sampler, 50 steps, CFG scale 5.0, class-name-only text prompt; nothing is fine-tuned.", "evidence": "ImageIDC/ImageNette, ResNet10-AP, 256x256 (Table 1): CoDA-on-SDXL beats prior SOTA by 2.6/2.7/5.5 points on ImageIDC and 2.4/0.9/3.6 on ImageNette at IPC 10/20/50 (e.g. ImageIDC IPC50: 77.6±0.6 vs SDXL+D4M 56.6±0.2 vs SDXL+MGD3 61.4±1.4 vs SDXL raw 45.1±1.8); MGD3 transplanted onto SDXL collapses relative to its ImageNet-trained-LDM version, evidencing that MGD3's success depended on the target-trained prior, not its guidance algorithm alone. ImageWoof (Table 2): CoDA beats the prior SOTA (Minimax/MGD3 on a target-trained model) at IPC>=50 (e.g. IPC50 ResNet-18: 61.2±0.9 vs Minimax 57.1±0.6) but trails slightly at IPC<=20 (IPC10 ResNetAP-10: 39.2±0.7 vs MGD3 40.4±1.9). ImageNet-1K soft-label protocol, ResNet-18 teacher/student (Table 4, shared RDED-style KD protocol from Sun et al. 2024): IPC10 44.3±0.1 (on par with Minimax 44.3±0.5, below IGD 46.2±0.6, +16.4 over D4M 27.9±0.2); IPC50 60.4±0.2, a new SOTA exceeding IGD 60.3±0.4, Minimax 58.6±0.3 and D4M 55.2±0.1. Ablation of components (Table 7, \"R\" vs \"G\"): the CPU-only Distribution Discovery output alone (real, selected/resized images, no diffusion sampling at all) already beats the prior SOTA on Places365 IPC10 (used to show zero-shot domain transfer: 47.0±1.6 SDXL+Ours vs 41.7±1.6 random vs 34.4±1.6 SDXL+K-means); adding Distribution Alignment (G) improves further (e.g. ImageIDC IPC50: R 75.7±1.9 to G 77.6±0.6). Step-by-step ablation on MGD3-as-base (Fig. 3) shows UMAP preprocessing alone gives a partial gain over raw K-means on SDXL, but only the full HDBSCAN discovery pipeline surpasses the ImageNet-trained MGD3 baseline. Cost: Distribution Discovery takes 2.2 GPU-hours for all 1,000 ImageNet-1K classes (CPU-bound after VAE encoding); Distribution Alignment adds negligible per-step overhead, generating a 224x224 image in 2.7s (25-step sampling can even outperform 50-step, at 1.3s per 256x256 image).", "limitations": "Stated (Appendix A.3): the two key Distribution Discovery hyperparameters (UMAP n_neighbors, HDBSCAN min_cluster_size) drift slightly across datasets since each has its own core-distribution density, so some per-dataset grid search is needed even though it is CPU-cheap; the authors flag automatic hyperparameter selection as future work. Observed: still trails target-trained diffusion methods (IGD, Minimax) at low IPC on hard fine-grained datasets (ImageWoof IPC<=20) and at ImageNet-1K IPC10, so the training-free/off-the-shelf claim does not yet dominate uniformly across budgets; representativeness is defined purely by density in VAE-latent space, inheriting whatever class-mixing the VAE's Gaussian prior induces (Fig. 1b-c show poor separability there relative to a target-trained classifier's feature space); the guidance is a single real-sample target per generated image for the entire trajectory, so diversity depends entirely on how many distinct representative samples Distribution Discovery finds, not on any diversity-specific guidance term (contrast with MGD3's explicit diversity stage or Minimax's minimax loss)." } }, { "id": "tgdd-2026", "type": "paper", "title": "TGDD: Trajectory Guided Dataset Distillation with Balanced Distribution", "short": "TGDD", "arxiv": "2512.02469", "url": null, "date": "2025-12-02", "venue": { "name": "AAAI 2026", "status": "conference" }, "authors": [ "Fengli Ran", "Xiao Pu", "Bo Liu", "Xiuli Bi", "Bin Xiao" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching", "trajectory-matching" ], "tier": "catalogued", "links": { "code": "https://github.com/FlyFinley/TGDD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Reformulates distribution matching as dynamic alignment along the model's training trajectory rather than a single static feature space, capturing how feature representations evolve during training and adding a distribution-constraint regularizer to reduce class overlap, reporting a 5.0-point accuracy gain on high-resolution benchmarks with no added optimization overhead across ten datasets.", "abstract": "Dataset distillation compresses large datasets into compact synthetic ones to reduce storage and computational costs. Among various approaches, distribution matching (DM)-based methods have attracted attention for their high efficiency. However, they often overlook the evolution of feature representations during training, which limits the expressiveness of synthetic data and weakens downstream performance. To address this issue, we propose Trajectory Guided Dataset Distillation (TGDD), which reformulates distribution matching as a dynamic alignment process along the model's training trajectory. At each training stage, TGDD captures evolving semantics by aligning the feature distribution between the synthetic and original dataset. Meanwhile, it introduces a distribution constraint regularization to reduce class overlap. This design helps synthetic data preserve both semantic diversity and representativeness, improving performance in downstream tasks. Without additional optimization overhead, TGDD achieves a favorable balance between performance and efficiency. Experiments on ten datasets demonstrate that TGDD achieves state-of-the-art performance, notably a 5.0% accuracy gain on high-resolution benchmarks.", "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ], "builds_on": [ "dm-2023" ] }, { "id": "optimizing-distributional-geometry-alignment-2025", "type": "paper", "title": "Optimizing Distributional Geometry Alignment with Optimal Transport for Generative Dataset Distillation", "arxiv": "2512.00308", "url": null, "date": "2025-11-29", "venue": { "name": "NeurIPS 2025", "status": "conference" }, "authors": [ "Xiao Cui", "Yulei Qin", "Wengang Zhou", "Hongsheng Li", "Houqiang Li" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis", "label-distillation" ], "facets": { "labels": "soft-relabel" }, "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Reformulates generative dataset distillation as Optimal Transport distance minimization to capture instance-level and intra-class geometry that global mean/variance matching misses, with three components: OT-guided diffusion sampling that aligns latent distributions, label-image-aligned soft relabeling that adapts label distributions to the complexity of the distilled images, and OT-based logit matching between student outputs and soft-label distributions; reports at least a 4% accuracy gain over the state of the art at IPC=10 on ImageNet-1K across architectures.", "abstract": "Dataset distillation seeks to synthesize a compact distilled dataset, enabling models trained on it to achieve performance comparable to models trained on the full dataset. Recent methods for large-scale datasets focus on matching global distributional statistics (e.g., mean and variance), but overlook critical instance-level characteristics and intraclass variations, leading to suboptimal generalization. We address this limitation by reformulating dataset distillation as an Optimal Transport (OT) distance minimization problem, enabling fine-grained alignment at both global and instance levels throughout the pipeline. OT offers a geometrically faithful framework for distribution matching. It effectively preserves local modes, intra-class patterns, and fine-grained variations that characterize the geometry of complex, high-dimensional distributions. Our method comprises three components tailored for preserving distributional geometry: (1) OT-guided diffusion sampling, which aligns latent distributions of real and distilled images; (2) label-image-aligned soft relabeling, which adapts label distributions based on the complexity of distilled image distributions; and (3) OT-based logit matching, which aligns the output of student models with soft-label distributions. Extensive experiments across diverse architectures and large-scale datasets demonstrate that our method consistently outperforms state-of-the-art approaches in an efficient manner, achieving at least 4% accuracy improvement under IPC=10 settings for each architecture on ImageNet-1K.", "awesome_sections": [ "Main / Generative Distillation / Diffusion" ] }, { "id": "algorithmic-guarantees-for-distilling-2026", "type": "paper", "title": "Algorithmic Guarantees for Distilling Supervised and Offline RL Datasets", "arxiv": "2512.00536", "url": null, "date": "2025-11-29", "venue": { "name": "ICLR 2026", "status": "conference" }, "authors": [ "Aaryan Gupta", "Rishi Saket", "Aravindan Raghuveer" ], "kind": "method", "settings": [ "other-data" ], "families": [ "distribution-matching" ], "facets": { "regime": "training-free", "prior": [ "random-networks" ] }, "applications": [ "rl" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Gives a dataset-distillation algorithm for linear regression that matches training and synthetic losses with respect to a fixed set of randomly sampled regressors, with no model training in the loop, proving it needs only O(d^2) sampled regressors for any bounded linear model's synthetic-data loss to track its real-data loss (with a matching Omega(d^2) lower bound), then extends the algorithm to offline reinforcement learning by matching Bellman loss instead of behavioral cloning, using both rewards and next-state information.", "abstract": "Given a training dataset, the goal of dataset distillation is to derive a synthetic dataset such that models trained on the latter perform as well as those trained on the training dataset. In this work, we develop and analyze an efficient dataset distillation algorithm for supervised learning, specifically regression in $\\mathbb{R}^d$, based on matching the losses on the training and synthetic datasets with respect to a fixed set of randomly sampled regressors without any model training. Our first key contribution is a novel performance guarantee proving that our algorithm needs only $\\tilde{O}(d^2)$ sampled regressors to derive a synthetic dataset on which the MSE loss of any bounded linear model is nearly the same as its MSE loss on the given training data. In particular, the model optimized on the synthetic data has close to minimum loss on the training data, thus performing nearly as well as the model optimized on the latter. Complementing this, we also prove a matching lower bound of $\\Omega(d^2)$ for the number of sampled regressors showing the tightness of our analysis. Our second contribution is to extend our algorithm to offline RL dataset distillation by matching the Bellman loss, unlike previous works which used a behavioral cloning objective. This is the first such method which leverages both, the rewards and the next state information, available in offline RL datasets, without any policy model optimization. Our algorithm generates a synthetic dataset whose Bellman loss with respect to any linear action-value predictor is close to the latter's Bellman loss on the offline RL training dataset. Therefore, a policy associated with an action-value predictor optimized on the synthetic dataset performs nearly as well as that derived from the one optimized on the training data. We conduct experiments to validate our theoretical guarantees and observe performance gains.", "awesome_sections": [ "Applications / Reinforcement Learning" ], "notes": "Family tagged as distribution-matching by analogy (random functions replace random networks); flagged as a classification doubt since the paper does not describe itself in P1-P4 terms." }, { "id": "multimodal-graph-gm-2025", "type": "paper", "title": "Decoupling and Damping: Structurally-Regularized Gradient Matching for Multimodal Graph Condensation", "short": "SR-GM", "arxiv": "2511.20222", "url": null, "date": "2025-11-25", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Lian Shen", "Zhendan Chen", "Meijia Song", "Yinhui jiang", "Ziming Su", "Juan Liu", "Xiangrong Liu" ], "kind": "method", "settings": [ "graph" ], "families": [ "gradient-matching" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Condenses multimodal graphs, where node features integrate sources such as vision and text, by decoupling per-modality gradients before matching so that semantic misalignment between modalities does not create gradient conflicts, and adds a structural damping regularizer that suppresses gradient noise the message-passing mechanism would otherwise amplify through the topology. Reports state-of-the-art results and cross-architecture generalization on four multimodal graph datasets.", "abstract": "In multimodal graph learning, graph structures that integrate information from multiple sources, such as vision and text, can more comprehensively model complex entity relationships. However, the continuous growth of their data scale poses a significant computational bottleneck for training. Graph condensation methods provide a feasible path forward by synthesizing compact and representative datasets. Nevertheless, existing condensation approaches generally suffer from performance limitations in multimodal scenarios, mainly due to two reasons: (1) semantic misalignment between different modalities leads to gradient conflicts; (2) the message-passing mechanism of graph neural networks further structurally amplifies such gradient noise. Based on this, we propose Structural Regularized Gradient Matching (SR-GM), a condensation framework for multimodal graphs. This method alleviates gradient conflicts between modalities through a gradient decoupling mechanism and introduces a structural damping regularizer to suppress the propagation of gradient noise in the topology, thereby transforming the graph structure from a noise amplifier into a training stabilizer. Extensive experiments on four multimodal graph datasets demonstrate the effectiveness of SR-GM, highlighting its state-of-the-art performance and cross-architecture generalization capabilities in multimodal graph dataset condensation.", "notes": "" }, { "id": "rethinking-long-tailed-dataset-2026", "type": "paper", "title": "Rethinking Long-tailed Dataset Distillation: A Uni-Level Framework with Unbiased Recovery and Relabeling", "short": "RLDD", "arxiv": "2511.18858", "url": null, "date": "2025-11-24", "venue": { "name": "AAAI 2026", "status": "conference" }, "authors": [ "Xiao Cui", "Yulei Qin", "Xinyue Li", "Wengang Zhou", "Hongsheng Li", "Houqiang Li" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis", "label-distillation" ], "facets": { "labels": "soft-relabel" }, "applications": [ "long-tail" ], "tier": "notable", "links": { "code": "https://github.com/2018cx/RLDD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Moves long-tailed dataset distillation away from trajectory matching to a statistical-alignment perspective, recalibrating Batch Normalization statistics via a full forward pass with dynamically adjusted momentum, using separate observer and teacher expert models for reliable statistics and soft-label generation, and initializing synthetic images by incrementally selecting high-confidence, diverse augmentations. Reports 15.6% top-1 accuracy improvement on CIFAR-100-LT and 11.8% on Tiny-ImageNet-LT at IPC=10, IF=10, over prior state-of-the-art long-tailed distillation methods.", "abstract": "Dataset distillation creates a small distilled set that enables efficient training by capturing key information from the full dataset. While existing dataset distillation methods perform well on balanced datasets, they struggle under long-tailed distributions, where imbalanced class frequencies induce biased model representations and corrupt statistical estimates such as Batch Normalization (BN) statistics. In this paper, we rethink long-tailed dataset distillation by revisiting the limitations of trajectory-based methods, and instead adopt the statistical alignment perspective to jointly mitigate model bias and restore fair supervision. To this end, we introduce three dedicated components that enable unbiased recovery of distilled images and soft relabeling: (1) enhancing expert models (an observer model for recovery and a teacher model for relabeling) to enable reliable statistics estimation and soft-label generation; (2) recalibrating BN statistics via a full forward pass with dynamically adjusted momentum to reduce representation skew; (3) initializing synthetic images by incrementally selecting high-confidence and diverse augmentations via a multi-round mechanism that promotes coverage and diversity. Extensive experiments on four long-tailed benchmarks show consistent improvements over state-of-the-art methods across varying degrees of class imbalance. Notably, our approach improves top-1 accuracy by 15.6% on CIFAR-100-LT and 11.8% on Tiny-ImageNet-LT under IPC=10 and IF=10. Codes are available at https://github.com/2018cx/RLDD.", "awesome_sections": [ "Applications / Long-Tail" ], "notes": "" }, { "id": "rectifying-soft-label-entangled-2025", "type": "paper", "title": "Rectifying Soft-Label Entangled Bias in Long-Tailed Dataset Distillation", "short": "ADSA", "arxiv": "2511.17914", "url": null, "date": "2025-11-22", "venue": { "name": "NeurIPS 2025", "status": "conference" }, "authors": [ "Chenyang Jiang", "Hang Zhao", "Xinyu Zhang", "Zhengcen Li", "Qiben Shan", "Shaocong Wu", "Jingyong Su" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "label-distillation" ], "facets": { "labels": "soft-relabel" }, "applications": [ "long-tail" ], "tier": "notable", "links": { "code": "https://github.com/j-cyoung/ADSA_DD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Derives an imbalance-aware generalization bound for models trained on distilled data and traces long-tailed dataset distillation's performance loss to soft-label bias entangled from two sources, the distillation model and the distilled images themselves, identified through systematic perturbation of imbalance levels; proposes ADSA, a lightweight Adaptive Soft-label Alignment module that calibrates this bias and plugs into existing distillation pipelines. Reports up to 11.8% tail-class accuracy improvement and 41.4% overall accuracy on ImageNet-1k-LT with EDC at IPC=50.", "abstract": "Dataset distillation compresses large-scale datasets into compact, highly informative synthetic data, significantly reducing storage and training costs. However, existing research primarily focuses on balanced datasets and struggles to perform under real-world long-tailed distributions. In this work, we emphasize the critical role of soft labels in long-tailed dataset distillation and uncover the underlying mechanisms contributing to performance degradation. Specifically, we derive an imbalance-aware generalization bound for model trained on distilled dataset. We then identify two primary sources of soft-label bias, which originate from the distillation model and the distilled images, through systematic perturbation of the data imbalance levels. To address this, we propose ADSA, an Adaptive Soft-label Alignment module that calibrates the entangled biases. This lightweight module integrates seamlessly into existing distillation pipelines and consistently improves performance. On ImageNet-1k-LT with EDC and IPC=50, ADSA improves tail-class accuracy by up to 11.8% and raises overall accuracy to 41.4%. Extensive experiments demonstrate that ADSA provides a robust and generalizable solution under limited label budgets and across a range of distillation techniques. Code is available at: https://github.com/j-cyoung/ADSA_DD.git.", "awesome_sections": [ "Applications / Long-Tail" ], "builds_on": [ "edc-2024" ], "notes": "" }, { "id": "davdd-2025", "type": "paper", "title": "Decoupled Audio-Visual Dataset Distillation", "short": "DAVDD", "arxiv": "2511.17890", "url": null, "date": "2025-11-22", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Wenyuan Li", "Guang Li", "Keisuke Maeda", "Takahiro Ogawa", "Miki Haseyama" ], "kind": "method", "settings": [ "audio-visual-omni" ], "families": [ "distribution-matching" ], "facets": { "prior": [ "trained-experts" ], "labels": "hard", "regime": "single-level", "parameterization": [ "pixels" ], "scale": [ "large-scale-other" ] }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-14", "source": "arxiv-api", "depth": "full-text" }, "summary": "Proposes DAVDD, a decoupled audio-visual distillation framework built on distribution matching: a bank of pretrained, frozen audio-visual encoder pairs feeds lightweight decoupler MLPs that split each modality's features into shared and private representations; a Common Intermodal Matching loss aligns the shared representations across modalities while a Sample-Distribution Joint Alignment combines instance-level contrastive alignment with EMA class prototypes, and private representations are kept out of the cross-modal loss entirely to protect modality-specific cues. On VGGS-10K at 10 images-per-class it reports 56.2% accuracy versus 54.0% for AVDD and 68.2% for the full dataset (Table 1).", "abstract": "Audio-Visual Dataset Distillation aims to compress large-scale datasets into compact subsets while preserving the performance of the original data. However, conventional Distribution Matching (DM) methods struggle to capture intrinsic cross-modal alignment. Subsequent studies have attempted to introduce cross-modal matching, but two major challenges remain: (i) independently and randomly initialized encoders lead to inconsistent modality mapping spaces, increasing training difficulty; and (ii) direct interactions between modalities tend to damage modality-specific (private) information, thereby degrading the quality of the distilled data. To address these challenges, we propose DAVDD, a pretraining-based decoupled audio-visual distillation framework. DAVDD leverages a diverse pretrained bank to obtain stable modality features and uses a lightweight decoupler bank to disentangle them into common and private representations. To effectively preserve cross-modal structure, we further introduce Common Intermodal Matching together with a Sample-Distribution Joint Alignment strategy, ensuring that shared representations are aligned both at the sample level and the global distribution level. Meanwhile, private representations are entirely isolated from cross-modal interaction, safeguarding modality-specific cues throughout distillation. Extensive experiments across multiple benchmarks show that DAVDD achieves state-of-the-art results under all IPC settings, demonstrating the effectiveness of decoupled representation learning for high-quality audio-visual dataset distillation. Code will be released.", "builds_on": [ "avdd-2024" ], "awesome_sections": [ "Main / Multimodal Distillation" ], "explained": { "before": "AVDD extended distribution matching to audio-visual data by summing per-modality DM losses and adding joint-matching and modality-gap-matching terms computed with randomly initialized visual/audio ConvNets (freshly re-sampled each iteration, never pretrained).", "problem": "The paper diagnoses two problems specific to AVDD's random-network recipe: (1) independently and randomly initialized visual and audio encoders leave the two modalities in mismatched embedding spaces to begin with, which the paper argues complicates cross-modal matching before it even starts; and (2) enforcing cross-modal (inter-modal) matching directly on the same features used for intra-modal matching creates conflicting objectives, since a feature that helps align across modalities is not necessarily the feature that best represents that modality alone, and AVDD's single shared representation has no way to serve both roles without compromise - degrading modality-private information.", "idea": "Use a bank of diverse, pretrained (not randomly initialized) audio-visual encoder pairs for stable, consistent feature spaces, and attach lightweight decoupler MLPs to each encoder that split its output into a shared subspace (used only for cross-modal matching) and a private subspace (used only for intra-modal matching), so the two objectives never compete for the same features; keep private representations entirely outside the cross-modal loss to protect modality-specific cues.", "method": "Synthetic 128x56 log-mel spectrograms (audio) and 224x224 video frames are directly optimized. A bank of $M{=}40$ pretrained encoder pairs (audio: 3-block ConvNet; visual: 5-block ConvNet), each independently trained on the full real dataset, is held frozen throughout distillation. Each encoder pair has $T{=}4$ decoupler instances (2-layer MLPs $g^a_{m,t}, g^v_{m,t}$ per modality) that are trained during distillation to project frozen-encoder features into a shared space $\\mathbb R^{d_c}$ (6272-d). Private-representation matching (Eq. 12-14) applies per-modality DM (mean-feature matching) independently to each modality's frozen-encoder features. Common Intermodal Matching (Eq. 15-19) applies DM-style matching to the decoupled shared representations, both per-modality and jointly summed across modalities (as in AVDD's joint-matching term, but now on decoupled shared features rather than raw encoder features). Sample-Distribution Joint Alignment (Eq. 22-27) adds an intra-sample symmetric NT-Xent contrastive loss treating true audio-visual pairs as positives, plus a distribution-level term that aligns batch-mean shared features to cross-modal EMA class prototypes ($P^A_c,P^V_c$) by cosine distance. The decoupler MLPs themselves are trained with a separate decoupling loss (Eq. 21, 28: classification on shared representations plus inter- and intra-sample contrastive terms) so they learn a meaningful shared/private split rather than an arbitrary one. At evaluation, a fresh ConvNet (or alternative architecture) is trained from scratch on the synthetic data only, as in AVDD.", "evidence": "Table 1 (5-run averages): VGGS-10K IPC1/10/20: DAVDD 41.9/56.2/59.1 vs AVDD 40.2/54.0/57.7 vs whole-data 68.2 (IPC10 column). MUSIC-21 IPC1/10/20: DAVDD 45.7/67.8/71.4 vs AVDD 44.2/66.9/69.7 vs whole-data 85.9. AVE IPC1/10/20: DAVDD 23.6/37.3/40.9 vs AVDD 22.7/35.9/40.0 vs whole-data 52.2. Gains over AVDD are consistently positive but modest: +0.9 to +2.2 points depending on dataset/IPC. Against the plain DM baseline the gap is larger (VGGS-10K IPC1: 41.9 vs 36.5, +5.4; MUSIC-21 IPC1: 45.7 vs 38.3, +7.4). Cross-architecture generalization (Table 4, VGGS-10K IPC1): DAVDD beats AVDD on ConvNet (41.9 vs 40.2), VGG11 (34.6 vs 34.0), and ResNet18 (31.0 vs 30.5) - all small margins. The key ablation (Table 2, VGGS-10K IPC10, additive from a DM baseline at 43.9): adding the pretrained encoder bank alone (in place of random encoders) reaches 49.2 (+5.3 over DM); adding the decoupler bank on top reaches 54.9 (+5.7 further, i.e. the largest single increment); adding Common Intermodal Matching reaches the full 56.2 (+12.3 total over DM). The pretrained-bank switch and the decoupling mechanism each contribute comparably large, roughly equal-sized gains. No GPU-hour, wall- clock or memory cost is reported anywhere in the paper (all experiments on one RTX A6000 Ada GPU).", "limitations": "The paper states no explicit limitations section. Gains over AVDD are consistently small (0.9-2.2 points) despite the added machinery of 40 pretrained encoder pairs and 160 total decoupler MLPs (4 per pair), and no training-time, memory, or wall-clock comparison against AVDD's cheaper random-network recipe is given, so the added cost of maintaining and running the pretrained encoder bank is not weighed against the accuracy gain. The paper's own t-SNE visualization (their Appendix Figure 8) shows the improvement is more visible for the visual modality than for audio. The number of encoder pairs (40) and decouplers per pair (4) are not ablated or theoretically justified, appearing to be empirical choices. Evaluation covers only audio-visual event recognition (classification), unlike AVDD, which also reports retrieval results - DAVDD's paper does not include a retrieval evaluation for direct comparison on that axis." }, "notes": "Table numbers were extracted from the arXiv HTML by automated summarization; recommend spot-checking Table 1 directly before publishing on the site." }, { "id": "linear-gradient-matching-2025", "type": "paper", "title": "Dataset Distillation for Pre-Trained Self-Supervised Vision Models", "short": "Linear Gradient Matching", "arxiv": "2511.16674", "url": null, "date": "2025-11-20", "venue": { "name": "NeurIPS 2025", "status": "conference" }, "authors": [ "George Cazenavette", "Antonio Torralba", "Vincent Sitzmann" ], "kind": "method", "settings": [ "self-supervised" ], "families": [ "gradient-matching" ], "facets": { "prior": [ "pretrained-classifier" ] }, "builds_on": [ "dc-2021", "dsa-2021", "mtt-2022", "glad-2023" ], "tier": "notable", "links": { "code": "https://github.com/GeorgeCazenavette/linear-gradient-matching", "project": "https://linear-gradient-matching.github.io/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Linear Gradient Matching distills a dataset for training linear probes on top of a frozen, pretrained self-supervised vision model, optimizing synthetic images so that gradients they induce in the linear classifier match those from real data. The paper reports that the resulting synthetic sets outperform all real-image baselines, generalize across pretrained backbones (e.g. a set distilled via DINO trains a competitive linear CLIP probe), and are effective for fine-grained classification and for interpretability probes such as measuring embedding-space similarity between models.", "abstract": "The task of dataset distillation aims to find a small set of synthetic images such that training a model on them reproduces the performance of the same model trained on a much larger dataset of real samples. Existing distillation methods focus on synthesizing datasets that enable training randomly initialized models. In contrast, state-of-the-art vision approaches are increasingly building on large, pre-trained self-supervised models rather than training from scratch. In this paper, we investigate the problem of distilling datasets that enable us to optimally train linear probes on top of such large, pre-trained vision models. We introduce a method of dataset distillation for this task called Linear Gradient Matching that optimizes the synthetic images such that, when passed through a pre-trained feature extractor, they induce gradients in the linear classifier similar to those produced by the real data. Our method yields synthetic data that outperform all real-image baselines and, remarkably, generalize across pre-trained vision models, enabling us, for instance, to train a linear CLIP probe that performs competitively using a dataset distilled via a DINO backbone. Further, we show that our distilled datasets are exceptionally effective for fine-grained classification and provide a valuable tool for model interpretability, predicting, among other things, how similar two models' embedding spaces are under the platonic representation hypothesis or whether a model is sensitive to spurious correlations in adversarial datasets.", "notes": "", "explained": { "before": "Essentially all prior dataset distillation (DC's gradient matching, MTT's trajectory matching, kernel-closed-form methods) targets training a randomly initialized model from scratch, and scaling those bi-level objectives to large models runs into memory limits and optimization instability (per the paper's reading of MTT and its extensions); meanwhile vision practice has shifted toward frozen, pretrained self-supervised backbones (CLIP, DINO-v2, EVA-02, MoCo-v3) with only a lightweight linear head trained on top, a regime existing distillation methods do not target.", "problem": "No prior method distills a dataset specifically for training a linear probe on top of a frozen pretrained feature extractor, and a naive port of gradient matching to this setting (matching gradients of the full model or of a randomly-initialized model) does not obviously transfer, since the object of interest is now only the linear classifier's gradient in a fixed embedding space, not a whole network's parameters.", "idea": "Restrict gradient matching to just the linear classifier: sample a fresh random linear layer at each distillation step, pass real and synthetic images through the frozen pretrained feature extractor and that linear layer, and optimize the synthetic images so the classification-loss gradient with respect to the linear layer matches between real and synthetic data — reducing dataset distillation, in this regime, to matching gradients of a single linear layer rather than an entire network.", "method": "Given a frozen self-supervised feature extractor $\\phi$, at each step sample $W\\sim\\mathcal{N}(0,1)^ {c\\times f}$, compute $\\ell_{real}=CE(W\\phi(X_{real}),Y_{real})$ and $\\ell_{syn}=CE(W\\phi(X_{syn}), Y_{syn})$, and minimize the meta loss $\\mathcal{L}_{meta}=1-\\cos(\\text{vec}(\\partial\\ell_{real}/ \\partial W),\\text{vec}(\\partial\\ell_{syn}/\\partial W))$, backpropagating through the inner gradient computation to update $X_{syn}$ (Eq. 1-2, Fig. 2). To prevent the resulting images from overfitting to the specific backbone used for distillation — which the paper finds happens with naive pixel optimization — three additions are used: (1) a multi-scale pyramid image parameterization (resolutions 1x1 to 256x256, rendered by bilinear upsampling and summing, optimized progressively from coarse to fine) rather than raw pixels; (2) a fixed decorrelated-color-space linear transform (as in feature- visualization work) to avoid color biases from the distilling model; (3) differentiable augmentations (flip, random-resized-crop, Gaussian noise) applied as multiple augmented copies per synthetic image per step, concatenated together in the loss. Stored artifact: synthetic images plus their (fixed) class labels; no generator or teacher network is kept after distillation. Feature extractor $\\phi$ stays frozen throughout; only $X_{syn}$ (and the ephemeral random $W$) are optimized.", "evidence": "Table 1 (ImageNet-100 and ImageNet-1k, 1 image/class, four backbones — CLIP, DINO-v2, EVA-02, MoCo-v3): Linear Gradient Matching beats Random, Centroid, and Nearest-Neighbor real-image baselines on every backbone/dataset; e.g. ImageNet-1k with DINO-v2: 75.0% vs. best real-image baseline (Centroids) 69.5%, and vs. full-dataset (1.28M images) 83.0% — reaching within 8 points of full-data accuracy from a single image per class. Table 2 (cross-model transfer): images distilled with one backbone train competitive linear probes on a different backbone's embeddings in most cells (e.g. DINO-v2-distilled ImageNet-1k evaluated on EVA-02: 65.4%), with one notable failure pair (CLIP ↔ MoCo-v3, attributed to poor model alignment, Sec. 4.6). Table 3 / Fig. 3, 5 (ablation, ImageNet-100): removing the pyramid representation causes the largest cross-model performance drop (high-frequency overfitting artifacts visibly appear in Fig. 5); removing color decorrelation has the smallest quantitative effect but the largest visual effect (oversaturated, incorrect colors); increasing the number of differentiable- augmentation rounds per step monotonically improves both same-model and cross-model performance (Fig. 3). Cost (Compute Budget, Appendix): distilling ImageNet-100 takes ~3 hours on 1 H200 GPU; ImageNet-1k takes ~12 hours on 4 H200 GPUs (limited to 3 augmentation rounds instead of 10 due to a data loading bottleneck); the paper notes trained linear probes converge in minutes vs. up to a full day training on the full dataset.", "limitations": "Explicitly memory- and data-loading-bound — ImageNet-1k experiments had to use fewer augmentation rounds (3 vs. 10) than smaller experiments due to a data-loading bottleneck; the bi-level optimization forces use of PyTorch's DistributedParallel rather than DistributedDataParallel, causing slowdowns at the large batch sizes used; only linear-probe evaluation is studied, not full fine-tuning of the backbone; one backbone pair (CLIP and MoCo-v3) transfers poorly to each other, an unexplained outlier the paper attributes to representational misalignment rather than a mechanism failure; all experiments use ViT-B-sized backbones at 224x224, leaving larger backbone scales untested." } }, { "id": "ddtime-2025", "type": "paper", "title": "DDTime: Dataset Distillation with Spectral Alignment and Information Bottleneck for Time-Series Forecasting", "short": "DDTime", "arxiv": "2511.16715", "url": null, "date": "2025-11-20", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Yuqi Li", "Kuiye Ding", "Chuanguang Yang", "Hao Wang", "Haoxuan Wang", "Huiran Duan", "Junming Liu", "Yingli Tian" ], "kind": "method", "settings": [ "time-series" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Proposes DDTime, a lightweight plug-in for time-series-forecasting dataset distillation that corrects autocorrelation-induced value-alignment bias with a frequency-domain alignment mechanism and increases synthetic-sample diversity with an information-bottleneck-inspired inter-sample regularizer; reports about 30% relative accuracy gains over prior distillation methods across 20 benchmark datasets and several forecasting architectures, at roughly 2.49% computational overhead.", "abstract": "Time-series forecasting is fundamental across many domains, yet training accurate models often requires large-scale datasets and substantial computational resources. Dataset distillation offers a promising alternative by synthesizing compact datasets that preserve the learning behavior of full data. However, extending dataset distillation to time-series forecasting is non-trivial due to two fundamental challenges: 1.temporal bias from strong autocorrelation, which leads to distorted value-term alignment between teacher and student models; and 2.insufficient diversity among synthetic samples, arising from the absence of explicit categorical priors to regularize trajectory variety. In this work, we propose DDTime, a lightweight and plug-in distillation framework built upon first-order condensation decomposition. To tackle Challenge 1, it revisits value-term alignment through temporal statistics and introduces a frequency-domain alignment mechanism to mitigate autocorrelation-induced bias, ensuring spectral consistency and temporal fidelity. To address Challenge 2, we further design an inter-sample regularization inspired by the information bottleneck principle, which enhances diversity and maximizes information density across synthetic trajectories. The combined objective is theoretically compatible with a wide range of condensation paradigms and supports stable first-order optimization. Extensive experiments on 20 benchmark datasets and diverse forecasting architectures demonstrate that DDTime consistently outperforms existing distillation methods, achieving about 30% relative accuracy gains while introducing about 2.49% computational overhead. All code and distilled datasets will be released.", "awesome_sections": [ "Applications / Time Series" ], "notes": "" }, { "id": "low-level-dataset-distillation-2025", "type": "paper", "title": "Low-Level Dataset Distillation for Medical Image Enhancement", "arxiv": "2511.13106", "url": null, "date": "2025-11-17", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Fengzhi Xu", "Ziyuan Yang", "Mengyu Sun", "Joey Tianyi Zhou", "Yi Zhang" ], "kind": "method", "settings": [ "dense-prediction" ], "families": [ "gradient-matching" ], "applications": [ "medical" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Argues that low-level tasks (denoising, enhancement) make dataset distillation underdetermined because the many-to-many pixel-level mapping cannot be constrained by a small distilled set the way many-to-one classification labels can; proposes the first low-level DD method, which builds a shared anatomical prior from one representative patient, personalizes it per patient with a Structure-Preserving Personalized Generation module, and injects patient-specific knowledge by matching gradients between networks trained on distilled pairs and on the patient's raw data. Downstream users only receive the abstracted distilled pairs, not raw patient data.", "abstract": "Medical image enhancement is clinically valuable, but existing methods require large-scale datasets to learn complex pixel-level mappings. However, the substantial training and storage costs associated with these datasets hinder their practical deployment. While dataset distillation (DD) can alleviate these burdens, existing methods mainly target high-level tasks, where multiple samples share the same label. This many-to-one mapping allows distilled data to capture shared semantics and achieve information compression. In contrast, low-level tasks involve a many-to-many mapping that requires pixel-level fidelity, making low-level DD an underdetermined problem, as a small distilled dataset cannot fully constrain the dense pixel-level mappings. To address this, we propose the first low-level DD method for medical image enhancement. We first leverage anatomical similarities across patients to construct the shared anatomical prior based on a representative patient, which serves as the initialization for the distilled data of different patients. This prior is then personalized for each patient using a Structure-Preserving Personalized Generation (SPG) module, which integrates patient-specific anatomical information into the distilled dataset while preserving pixel-level fidelity. For different low-level tasks, the distilled data is used to construct task-specific high- and low-quality training pairs. Patient-specific knowledge is injected into the distilled data by aligning the gradients computed from networks trained on the distilled pairs with those from the corresponding patient's raw data. Notably, downstream users cannot access raw patient data. Instead, only a distilled dataset containing abstract training information is shared, which excludes patient-specific details and thus preserves privacy.", "awesome_sections": [ "Applications / Medical" ], "notes": "" }, { "id": "learning-from-dense-events-2025", "type": "paper", "title": "Learning from Dense Events: Towards Fast Spiking Neural Networks Training via Event Dataset Distillation", "short": "PACE", "arxiv": "2511.12095", "url": null, "date": "2025-11-15", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Shuhan Ye", "Yi Yu", "Qixin Zhang", "Chenqi Kong", "Qiangqiang Wu", "Kun Wang", "Xudong Jiang" ], "kind": "method", "settings": [ "other-data" ], "families": [ "distribution-matching" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "PACE (Phase-Aligned Condensation for Events) is the first dataset distillation framework for spiking neural networks and event-based vision; its ST-DSM module uses residual membrane potentials to densify spike-based features and match amplitude and phase between synthetic and real event streams fine-grained in space and time, while a plug-and-play quantizer (PEQ-N) keeps the output compatible with standard event-frame pipelines. On N-MNIST the paper reports 84.4% accuracy (about 85% of full-data training) with more than 50x faster training and 6000x less storage, and stronger gains than coreset selection and other distillation baselines on DVS-Gesture and CIFAR10-DVS.", "abstract": "Event cameras sense brightness changes and output binary asynchronous event streams, attracting increasing attention. Their bio-inspired dynamics align well with spiking neural networks (SNNs), offering a promising energy-efficient alternative to conventional vision systems. However, SNNs remain costly to train due to temporal coding, which limits their practical deployment. To alleviate the high training cost of SNNs, we introduce \\textbf{PACE} (Phase-Aligned Condensation for Events), the first dataset distillation framework to SNNs and event-based vision. PACE distills a large training dataset into a compact synthetic one that enables fast SNN training, which is achieved by two core modules: \\textbf{ST-DSM} and \\textbf{PEQ-N}. ST-DSM uses residual membrane potentials to densify spike-based features (SDR) and to perform fine-grained spatiotemporal matching of amplitude and phase (ST-SM), while PEQ-N provides a plug-and-play straight through probabilistic integer quantizer compatible with standard event-frame pipelines. Across DVS-Gesture, CIFAR10-DVS, and N-MNIST datasets, PACE outperforms existing coreset selection and dataset distillation baselines, with particularly strong gains on dynamic event streams and at low or moderate IPC. Specifically, on N-MNIST, it achieves \\(84.4\\%\\) accuracy, about \\(85\\%\\) of the full training set performance, while reducing training time by more than \\(50\\times\\) and storage cost by \\(6000\\times\\), yielding compact surrogates that enable minute-scale SNN training and efficient edge deployment.", "notes": "" }, { "id": "prism-2026", "type": "paper", "title": "PRISM: Diversifying Dataset Distillation by Decoupling Architectural Priors", "short": "PRISM", "arxiv": "2511.09905", "url": null, "date": "2025-11-13", "venue": { "name": "TMLR 2026", "status": "journal" }, "authors": [ "Brian B. Moser", "Shalini Sarode", "Federico Raue", "Stanislav Frolov", "Krzysztof Adamkiewicz", "Arundhati Shanbhag", "Joachim Folz", "Tobias C. Nauen", "Andreas Dengel" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "facets": { "labels": "soft-relabel", "prior": [ "pretrained-classifier" ], "regime": "decoupled", "parameterization": [ "pixels" ], "scale": [ "imagenet-1k" ] }, "tier": "catalogued", "links": { "code": "https://github.com/Brian-Moser/prism", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "SRe2L uses one teacher for both the logit-matching loss and the BN-statistics regularizer. G-VBSM and EDC diversify by matching several backbones' BN statistics jointly (intra-distillation re-sampling, intra-class batches) but keep the two objectives coupled to whichever teacher is sampled at each step. D3S rotates an ensemble of same-architecture ResNet-18 teachers; CV-DD votes across teachers to build soft labels. RDED sidesteps inversion with real-crop selection; DELT splits IPC budget into early/late-optimized subtasks within a class.", "problem": "Every synthetic image in a class is optimized against the same global BN-statistics target from one teacher (or, in G-VBSM/EDC, the same *pooled* signal from several teachers used jointly), so the resulting images inherit that teacher's specific inductive bias and converge toward homogeneous, overly smooth solutions -- visible as high intra-class cosine similarity (0.86-0.92 for SRe2L/G-VBSM/DELT, Fig. 4) and confirmed qualitatively (Fig. 5: SRe2L images within a class converge to similar colors/textures from the same real-image initialization PRISM also uses).", "idea": "Split the two terms of the SRe2L objective -- the logit-matching classification loss and the BN-statistics regularizer -- and supervise them with architecturally *different* teacher models, rather than one model doing both; generalize the BN term to a randomly-sampled subset of several diverse teacher architectures per image, so the images are pulled toward \"natural\" statistics from multiple distinct architectural priors at once instead of one.", "method": "Follows the SRe2L recovery objective $X_S^*=\\arg\\min_{X_S}\\ell(\\varphi_\\phi(X_s),Y_s)+\\lambda \\mathcal{R}_{\\text{reg}}$ but decouples $\\phi$ (the logit/gradient teacher) from $\\theta$ (the BN teacher): dual-teacher decoupling uses two distinct models; the full PRISM generalizes the BN term to $\\mathbb{E}_{\\mathcal{M}_{\\text{sub}}\\sim P(\\mathbb{M}_{\\text{valid}})}[\\lambda\\sum_{\\omega\\in \\mathcal{M}_{\\text{sub}}}\\mathcal{R}_{\\text{BN}}^\\omega(X_s)]$, sampling a random subset (up to $k_{\\max}{=}4$, VRAM-limited) of a teacher pool (ResNet-18/34, ShuffleNetV2-0.5, MobileNetV2, EfficientNet-B0, later swapped for AlexNet) per image. A pre-distillation teacher-selection strategy (fixed teacher set per image before optimization begins) is shown to beat intra-distillation resampling (Table 2). Initialization uses one real image per synthetic image (no multi-image stitching as in RDED/EDC, no coreset selection as in DELT); cross-class batches (one IPC index per class per batch, following SRe2L) rather than intra-class batches (G-VBSM/EDC/DELT), which the authors argue keeps synthesis simple and parallelizable across GPUs -- diversity comes from architectural decoupling, not intra-batch diversity terms. Post-recovery: an MSE-based ensemble-relabeling loss with 0.1 ground-truth mixing, batch size 50 for relabeling, and EDC's SSRS cosine LR schedule ($\\zeta{=}2.5$).", "evidence": "ImageNet-1K, PRISM's own evaluation protocol (Table 1, mean of 3 seeds): ResNet-18, IPC=10 49.4% vs EDC 48.6%, RDED 42.0%, G-VBSM 31.4%, and PRISM's own re-run of SRe2L 21.3% (identical to SRe2L's own-reported value, unlike DELT/CIM/CV-DD's re-runs of the same nominal cell, which range 31.1-43.1%); IPC=50 PRISM 59.0% vs EDC 58.0%; IPC=100 PRISM 60.9%. ResNet-101, IPC=100: PRISM 68.6% vs EDC's IPC=50 value 64.9% (EDC does not report ResNet-101 IPC=100). Under DELT's evaluation protocol (Table 1, marked $\\ddagger$): ResNet-50, IPC=100, PRISM 69.4% vs DELT's own 66.1% (at IPC=50); ResNet-101, IPC=100, PRISM 70.4% vs DELT 67.6%. Diversity metric (Fig. 4): mean intra-class cosine similarity for PRISM is below 0.83, versus 0.86-0.92 for SRe2L/G-VBSM/DELT, using a pretrained ResNet-18 feature extractor. Recovery-only, no-KD diversity analysis (Table 2, ImageNet-1K, IPC on the order of the full per-class image count, ResNet-18, VRAM at synthesis batch size 100): SRe2L 17.9% vs a real-data upper bound of 70.0%; PRISM's full multi-teacher, pre-distillation, 4-BN-teacher configuration reaches 39.1% at 32.5 GB VRAM, versus 19.0% for dual-teacher decoupling alone (13.0 GB) and 18.3% for single-teacher intra-distillation alignment -- isolating architectural decoupling itself (not KD, not the LR schedule) as the source of the gain in a setting stripped of post-recovery optimization. No hard-label result and no ImageNet-21K or CIFAR/Tiny-ImageNet result are reported; the authors state this choice is deliberate since PRISM's contribution is only visible at large class count and scale.", "limitations": "VRAM scales linearly with the number of simultaneous BN teachers (6.5 GB for 1 teacher to 32.5 GB for 4, Table 2), capping how many architectural priors can be combined per image on a single GPU; no wall-clock synthesis time or GPU-hour comparison to SRe2L/G-VBSM/EDC is reported, only VRAM. The method still depends entirely on BN-statistics inversion and is not extended to normalization- free or LayerNorm/GroupNorm architectures (stated limitation). Cross-architecture generalization to unseen student backbones (ViT, ConvNeXt) is explicitly not tested, on the argument that PRISM's own teacher pool already spans architectures -- an argument, not a measurement. No hard-label (no-relabel) result is reported anywhere, so the accuracy gains cannot be separated from PRISM's own ensemble-relabeling scheme. Restricted to ImageNet-1K; no small-scale (CIFAR/Tiny-ImageNet) or ImageNet-21K results are given." }, "summary": "Argues that decoupled distillation inherits the inductive bias of whichever teacher supervises it, over-smoothing intra-class diversity as dataset size grows; PRISM decouples logit-matching from batch-normalization-alignment supervision, assigning them to different teacher architectures (a primary model for logits, a stochastic subset for BN alignment), with a scalable cross-class batch-formation scheme for fast parallel synthesis; on ImageNet-1K it reproducibly outperforms single-teacher SRe2L and multi-teacher G-VBSM at low and mid IPC with measurably richer intra-class diversity.", "abstract": "Dataset distillation (DD) promises compact yet faithful synthetic data, but existing approaches often inherit the inductive bias of a single teacher model. As dataset size increases, this bias drives generation toward overly smooth, homogeneous samples, reducing intra-class diversity and limiting generalization. We present PRISM (PRIors from diverse Source Models), a framework that disentangles architectural priors during synthesis. PRISM decouples the logit-matching and regularization objectives, supervising them with different teacher architectures: a primary model for logits and a stochastic subset for batch-normalization (BN) alignment. On ImageNet-1K, PRISM consistently and reproducibly outperforms single-teacher methods (e.g., SRe2L) and recent multi-teacher variants (e.g., G-VBSM) at low- and mid-IPC regimes. The generated data also show significantly richer intra-class diversity, as reflected by a notable drop in cosine similarity between features. We further analyze teacher selection strategies (pre- vs. intra-distillation) and introduce a scalable cross-class batch formation scheme for fast parallel synthesis. Code will be released after the review period.", "builds_on": [ "sre2l-2023", "g-vbsm-2024" ], "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "dp-geng-2026", "type": "paper", "title": "DP-GENG: Differentially Private Dataset Distillation Guided by DP-Generated Data", "short": "DP-GENG", "arxiv": "2511.09876", "url": null, "date": "2025-11-13", "venue": { "name": "AAAI 2026", "status": "conference" }, "authors": [ "Shuo Shi", "Jinghuai Zhang", "Shijie Jiang", "Chunyi Zhou", "Yuyuan Li", "Mengying Zhu", "Yangyang Wu", "Tianyu Du" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "applications": [ "privacy" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Initializes the distilled dataset with DP-generated data for realism, then refines it under a small privacy budget with a DP-feature-matching (distribution-matching) objective and aligns examples to their class distribution with an expert model, alongside a privacy-budget allocation strategy with a theoretical DP guarantee. Reports outperforming prior differentially-private dataset-distillation methods in both utility and robustness to membership-inference attacks.", "abstract": "Dataset distillation (DD) compresses large datasets into smaller ones while preserving the performance of models trained on them. Although DD is often assumed to enhance data privacy by aggregating over individual examples, recent studies reveal that standard DD can still leak sensitive information from the original dataset due to the lack of formal privacy guarantees. Existing differentially private (DP)-DD methods attempt to mitigate this risk by injecting noise into the distillation process. However, they often fail to fully leverage the original dataset, resulting in degraded realism and utility. This paper introduces \\libn, a novel framework that addresses the key limitations of current DP-DD by leveraging DP-generated data. Specifically, \\lib initializes the distilled dataset with DP-generated data to enhance realism. Then, generated data refines the DP-feature matching technique to distill the original dataset under a small privacy budget, and trains an expert model to align the distilled examples with their class distribution. Furthermore, we design a privacy budget allocation strategy to determine budget consumption across DP components and provide a theoretical analysis of the overall privacy guarantees. Extensive experiments show that \\lib significantly outperforms state-of-the-art DP-DD methods in terms of both dataset utility and robustness against membership inference attacks, establishing a new paradigm for privacy-preserving dataset distillation.", "awesome_sections": [ "Applications / Privacy" ], "notes": "" }, { "id": "imagebinddc-2026", "type": "paper", "title": "ImagebindDC: Compressing Multi-modal Data with Imagebind-based Condensation", "short": "ImageBindDC", "arxiv": "2511.08263", "url": null, "date": "2025-11-11", "venue": { "name": "AAAI 2026", "status": "conference" }, "authors": [ "Yue Min", "Shaobo Wang", "Jiaze Li", "Tianle Niu", "Junxin Fan", "Yongliang Miao", "Lijin Yang", "Linfeng Zhang" ], "kind": "method", "settings": [ "audio-visual-omni" ], "families": [ "distribution-matching" ], "facets": { "labels": "hard", "prior": [ "pretrained-classifier" ], "regime": "single-level", "parameterization": [ "pixels" ], "scale": [ "large-scale-other" ] }, "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-14", "source": "arxiv-api", "depth": "full-text" }, "summary": "ImageBindDC condenses omnimodal data inside ImageBind's shared embedding space using a Characteristic Function loss that performs exact infinite-moment matching in the Fourier domain, enforcing uni-modal, cross-modal (real-synthetic pair) and joint-modal distributional consistency simultaneously rather than matching each modality separately. On NYU-v2 the paper reports lossless performance versus full-data training at 5 condensed datapoints per class, an 8.2 percentage-point absolute improvement over the previous best method, and over 4x less condensation time.", "abstract": "Data condensation techniques aim to synthesize a compact dataset from a larger one to enable efficient model training, yet while successful in unimodal settings, they often fail in multimodal scenarios where preserving intricate inter-modal dependencies is crucial. To address this, we introduce ImageBindDC, a novel data condensation framework operating within the unified feature space of ImageBind. Our approach moves beyond conventional distribution-matching by employing a powerful Characteristic Function (CF) loss, which operates in the Fourier domain to facilitate a more precise statistical alignment via exact infinite moment matching. We design our objective to enforce three critical levels of distributional consistency: (i) uni-modal alignment, which matches the statistical properties of synthetic and real data within each modality; (ii) cross-modal alignment, which preserves pairwise semantics by matching the distributions of hybrid real-synthetic data pairs; and (iii) joint-modal alignment, which captures the complete multivariate data structure by aligning the joint distribution of real data pairs with their synthetic counterparts. Extensive experiments highlight the effectiveness of ImageBindDC: on the NYU-v2 dataset, a model trained on just 5 condensed datapoints per class achieves lossless performance comparable to one trained on the full dataset, achieving a new state-of-the-art with an 8.2\\% absolute improvement over the previous best method and more than 4$\\times$ less condensation time.", "builds_on": [ "avdd-2024", "lors-2024", "repblend-2025" ], "explained": { "before": "AVDD (and DAVDD) match feature distributions in separate, modality-specific spaces built from audio/visual-only backbones; LoRS reduces the full image-text relationship to a single learned scalar similarity per pair; RepBlend heuristically blends representations to fight modality collapse without a guarantee of preserving joint-modal semantics. Unimodal methods (DM, DSA, MTT) have no mechanism at all for keeping a specific sample's modalities linked together.", "problem": "The paper diagnoses that matching each modality's distribution independently (as unimodal methods and AVDD-style per-modality DM do) \"critically fails to capture the cross-modal relationships that link the data together\" - it can produce synthetic data that is statistically representative within each modality while breaking the pairing between, say, a specific image and its corresponding sound; and that kernel-based statistics like MMD (used by DM/AVDD) only approximate a finite number of moments, leaving room for distributional mismatch that a higher-fidelity statistic could catch.", "idea": "Condense inside ImageBind's single, already-aligned joint embedding space (rather than separate per-modality spaces built from scratch), and replace MMD-style feature-mean matching with a Characteristic Function loss that matches the full (Fourier-domain) characteristic function of real versus synthetic embedding distributions - equivalent to matching infinitely many moments rather than just means/variances - applied simultaneously at three levels: within each modality, between real-synthetic hybrid pairs across modalities, and over the full joint multi-modal distribution.", "method": "Synthetic raw data per modality (image pixels, raw audio waveforms, depth maps, text representations) are directly optimized. A frozen, pretrained ImageBind encoder maps every modality into a shared 1024-d embedding space; no encoder is trained during distillation. The Characteristic Function Discrepancy (CFD, Eq. 3) computes an L2 distance between the empirical characteristic functions (Fourier transforms of the embedding distributions) of real versus synthetic embeddings, capturing magnitude and phase information rather than only a kernel-weighted mean difference as MMD does. Three losses use CFD: uni-modal alignment (Eq. 5-7) applies CFD separately per modality ($\\mathcal L_{uni}=CFD(e_a,\\tilde e_a)+CFD(e_v,\\tilde e_v)$); cross-modal alignment (Eq. 8-11) forms real and synthetic hybrid pair products ($e_a\\odot e_v$ vs. $\\tilde e_a\\odot\\tilde e_v$) and matches them by cosine similarity ($\\mathcal L_{cross}=1-\\rho_{cross}$); joint-modal alignment (Eq. 12-13) forms batch-mean cross-products of the two modalities' means and matches them similarly ($\\mathcal L_{joint}=1-\\rho_{joint}$). The total loss (Eq. 14) is a weighted sum $\\lambda_{uni}\\mathcal L_{uni}+\\lambda_{cross}\\mathcal L_{cross}+\\lambda_{joint}\\mathcal L_{joint}$ (all weights fixed at 1.0 in the paper's experiments). At evaluation, a classifier is trained from scratch on the condensed data (with ImageBind's encoder still frozen as the feature extractor); no classifier is trained during the condensation phase itself.", "evidence": "NYU-v2 depth-text classification (Table 3): ImageBindDC 80.43/97.30/98.73% at 1/5/10 datapoints-per-class (DPC) vs AVDD 72.22/95.92/98.62% and DM 67.97/89.08/96.89%, vs a 98.62% full-data ceiling - lossless performance is reached at 5 DPC (97.30% vs 98.62% full-data), and the 8.2-point headline gain is 98.73% (ImageBindDC@10 DPC) minus DM's 96.89% at the same budget (not versus AVDD, whose gap at 10 DPC is only 0.11 points). VGGS-10K (Table 2, ImageBind-guided condensation) at 10 DPC: ImageBindDC 56.11% vs AVDD 48.89%, DM 45.46%, vs a 60.57% full-data ceiling. AVE (Table 1) at 20 DPC: ImageBindDC 52.20% vs AVDD 38.41%, MTT 32.57%, herding 33.04%. Clotho audio-text retrieval (Table 4, 20 DPC): A2T@1 ImageBindDC 0.0362 vs AVDD 0.0316 vs DM 0.0268 vs full-data 0.0526; T2A@5 ImageBindDC 0.1567 vs AVDD 0.1330 vs DM 0.1148 vs full-data 0.1674. Cost (Table 6, VGGS-10K, single A100): at 20 DPC, ImageBindDC takes 123.74s/13.39GB versus AVDD's 700.1s/14.24GB (about 5.7x faster) and DM's 707.21s/14.24GB; the paper's \"4x less condensation time\" and \"75% memory savings\" headline figures are computed against a full-data training-epoch baseline (419.9s) rather than against AVDD specifically. The key ablations show: (Figure 3) the CF loss beats MMD-based DM matching consistently across settings (e.g. 1 DPC audio-only: 32.33% vs 27.87%; 10 DPC audio+video: 70.34% vs 69.26%); (Figure 4, AVE, additive from uni-modal-only 70.34%) adding joint-modal reaches ~71.5%, adding cross-modal reaches ~72.5%, and all three combined reach 73.67% (a 3.33-point gain over uni-modal alone) - the paper explicitly notes the three terms are synergistic rather than simply additive.", "limitations": "The paper states no explicit limitations section. The entire framework depends on a frozen, pretrained ImageBind encoder, so its applicability to modalities or domains poorly represented in ImageBind's own pretraining (e.g. specialized medical imagery or rare audio domains) is untested. Despite being framed as \"omnimodal,\" the reported experiments test at most 2-3 modalities jointly per benchmark (audio+video, depth+text, audio+text) - true joint condensation over 4+ modalities simultaneously is not demonstrated. Loss weights ($\\lambda_{uni},\\lambda_{cross},\\lambda_{joint}$) are fixed at 1.0 throughout and not individually ablated. The paper's own cross-architecture generalization table (Table 5) reports a substantially lower number for at least one architecture/budget combination than the same architecture achieves elsewhere in the paper, suggesting a possible overfitting-to-ImageBind's-embedding-structure effect that the paper does not discuss. The joint-modal alignment's specific matrix-multiplication-of-means construction (Eq. 12) is not justified against alternative joint-distribution statistics (e.g. joint CFD, sliced Wasserstein distance). Only qualitative (not quantitative, e.g. no LPIPS/FID) evidence is given for the claim that distilled images preserve better visual/semantic coherence than AVDD's." }, "notes": "Setting corrected from the draft's vision-language to audio-visual-omni: ImageBind spans more than two modalities (image/depth/text/audio/…), matching the omnimodal case in docs/taxonomy.md, and the setting file already lists this id as a key paper for audio-visual-omni." }, { "id": "covmatch-2025", "type": "paper", "title": "CovMatch: Cross-Covariance Guided Multimodal Dataset Distillation with Trainable Text Encoder", "short": "CovMatch", "arxiv": "2510.18583", "url": null, "date": "2025-10-21", "venue": { "name": "NeurIPS 2025", "status": "conference" }, "authors": [ "Yongmin Lee", "Hye Won Chung" ], "kind": "method", "settings": [ "vision-language" ], "families": [ "distribution-matching" ], "facets": { "labels": "hard", "prior": [ "pretrained-classifier" ], "regime": "single-level", "parameterization": [ "pixels", "embedding-features" ], "scale": [ "flickr30k-coco" ] }, "tier": "notable", "links": { "code": "https://github.com/Yongalls/CovMatch", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Proposes CovMatch for image-text dataset distillation, which fixes the encoders at each distillation step and matches the cross-covariance between real and synthetic image and text feature matrices, together with per-modality feature-distribution regularization, instead of unrolling trajectories; unlike prior methods that freeze the text tower, CovMatch keeps a BERT-base text encoder trainable end to end (only its embedding layer is frozen) alongside a pretrained NFNet image encoder. On Flickr30K at 500 pairs it reports 38.4% mean recall (Table 2), a 6.8-point absolute gain over the LoRS baseline (31.6%), and 19.6% mean recall on COCO at 500 pairs, a 6.1-point gain over LoRS.", "abstract": "Multimodal dataset distillation aims to synthesize a small set of image-text pairs that enables efficient training of large-scale vision-language models. While dataset distillation has shown promise in unimodal tasks, extending it to multimodal contrastive learning presents key challenges: learning cross-modal alignment and managing the high computational cost of large encoders. Prior approaches address scalability by freezing the text encoder and update only the image encoder and text projection layer. However, we find this severely limits semantic alignment and becomes a bottleneck for performance scaling. We propose CovMatch, a scalable dataset distillation framework that aligns the cross-covariance of real and synthetic features while regularizing feature distributions within each modality. Unlike prior approaches, CovMatch enables joint optimization of both encoders, leading to stronger cross-modal alignment and improved performance. Evaluated on Flickr30K and COCO, CovMatch outperforms state-of-the-art multimodal distillation methods and achieves up to 6.8% absolute gains in retrieval accuracy using only 500 synthetic pairs.", "builds_on": [ "lors-2024" ], "awesome_sections": [ "Main / Multimodal Distillation" ], "explained": { "before": "MTT-VL and LoRS both apply trajectory matching to image-text pairs but freeze the pretrained text encoder during distillation, training only a projection layer and (in LoRS) the image encoder, following the multimodal-DD convention set by MTT-VL that a full text-encoder trajectory is too costly to match.", "problem": "The paper diagnoses that a frozen text encoder plus a linear projection lacks the expressiveness to achieve real cross-modal semantic alignment: captions of the same image do not cluster tightly in the resulting embedding space (their Figure 2), and retrieval performance saturates and even degrades below random sampling as the pair budget grows past about 1,000 pairs under this constraint.", "idea": "Make both encoders trainable, but sidestep the cost of unrolled trajectory matching (which made a trainable text encoder computationally prohibitive, ~120GB/132 GPU-hours for multimodal MTT) by fixing both encoders within each distillation step and matching the cross-covariance of image and text features (plus per-modality feature regularization) instead of matching multi-step parameter trajectories; under a high-temperature linearization of the contrastive loss the per-step objective becomes closed-form, removing the need for unrolled backpropagation through training.", "method": "Synthetic image pixels and text token embeddings are directly optimized. A pretrained NFNet image encoder and a pretrained BERT-base text encoder are held fixed within each distillation step but periodically updated with one gradient step on real data (and reset to pretrained weights every T=50 steps); their projection heads are optimized implicitly through the closed-form per-step solution rather than matched via an unrolled trajectory. The loss (Eq. 13) is $\\mathcal L^{\\text{CovMatch}}=\\mathcal L^{\\text{cov}}+\\lambda(\\mathcal L^{\\text{feat}}_v+\\mathcal L^{\\text{feat}}_l)$: a cross-covariance alignment term $\\|\\rho\\cdot C^{\\mathcal T}-C^{\\mathcal S}\\|_F^2$ matching the cross-covariance between image and text features of real (T) versus synthetic (S) data (with a scale correction $\\rho$ for the size mismatch between the real and synthetic sets), plus an L2 term matching per-modality mean projected features between real and synthetic data. At evaluation, both encoders and their projections are fine-tuned end to end on the synthetic set for 100 epochs.", "evidence": "NFNet image encoder, BERT-base text encoder (trainable). Flickr30K@100 (Table 2): IR@1/5/10 10.1/28.6/40.9, TR@1/5/10 14.8/38.0/50.6, mean recall 30.5, vs LoRS re-run 8.3/24.1/35.1, 11.8/35.8/49.2, mean 27.4. @500: IR 14.7/38.4/51.4, TR 19.9/46.7/59.5, mean recall 38.4, vs LoRS re-run 10.0/28.9/41.6, 15.5/39.8/53.7, mean 31.6 (a 6.8-point absolute gain). COCO@500: IR 5.4/18.0/28.2, TR 8.1/23.5/34.6, mean recall 19.6, vs LoRS re-run mean 13.5 (a 6.1-point gain). Baselines (random, herding, k-center, MTT-VL, LoRS) are re-run under the same protocol, averaged over 5 runs. Cost (Table 1): the multimodal-MTT baseline needs 132 GPU-hours of expert training plus 120GB expert-trajectory storage plus 71GB distillation memory; CovMatch needs 0 hours of expert preparation, 0GB storage, 15GB distillation memory, 1.2s/iteration. The key ablations (Table 9, Figure 5c) isolate two mechanisms: freezing the text encoder during distillation (i.e. reverting to the LoRS-style protocol) drops Flickr30K@500 mean recall from 38.4 to 29.4 (about a 29% relative loss); and disabling the \"online model update\" (periodically refreshing the fixed encoders with a real-data gradient step) costs about 6 points, with updating on synthetic data instead giving worse results than either. A cross-architecture ablation (Table 3, distilled on NFNet+BERT, evaluated on NF-ResNet/NF-RegNet/ViT image encoders and DistilBERT) shows CovMatch reaches 15.5-16.1 average recall on unseen encoders versus 7-9 for MTT-VL/LoRS.", "limitations": "The method assumes pretrained image and text encoders are available in the first place and does not address distilling data for from-scratch pretraining. The closed-form step relies on a high-temperature linearization of the contrastive loss, an approximation that may diverge from the low-temperature InfoNCE ($\\tau\\le0.07$) used in practice. The periodic reinitialization/online update of the \"fixed\" encoders introduces some instability, since the synthetic data is optimized against a moving target. Absolute retrieval remains far below full-data training even at 500 pairs (COCO IR@1 5.4% vs a full-data ceiling around 25.1% reported in the paper's own table)." }, "notes": "" }, { "id": "dap-2026", "type": "paper", "title": "Diffusion Models as Dataset Distillation Priors", "short": "DAP", "arxiv": "2510.17421", "url": null, "date": "2025-10-20", "venue": { "name": "ICLR 2026", "status": "conference" }, "authors": [ "Duo Su", "Huyu Wu", "Huanran Chen", "Yiming Shi", "Yuzhu Wang", "Xi Ye", "Jun Zhu" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "prior": [ "diffusion", "text-to-image-diffusion" ], "regime": "training-free", "parameterization": [ "pixels" ], "scale": [ "imagenet-subsets", "imagenet-1k" ] }, "tier": "notable", "links": { "code": null, "project": "https://suduo94.github.io/Diffusion-As-Priors" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Formalizes the representativeness already implicit in a pretrained diffusion model by quantifying feature-space similarity between synthetic and real data with a Mercer kernel, then uses it as training-free guidance for the reverse diffusion process instead of adding external constraints; reports state-of-the-art fidelity and cross-architecture generalization on ImageNet-1K and its subsets.", "abstract": "Dataset distillation aims to synthesize compact yet informative datasets from large ones. A significant challenge in this field is achieving a trifecta of diversity, generalization, and representativeness in a single distilled dataset. Although recent generative dataset distillation methods adopt powerful diffusion models as their foundation models, the inherent representativeness prior in diffusion models is overlooked. Consequently, these approaches often necessitate the integration of external constraints to enhance data quality. To address this, we propose Diffusion As Priors (DAP), which formalizes representativeness by quantifying the similarity between synthetic and real data in feature space using a Mercer kernel. We then introduce this prior as guidance to steer the reverse diffusion process, enhancing the representativeness of distilled samples without any retraining. Extensive experiments on large-scale datasets, such as ImageNet-1K and its subsets, demonstrate that DAP outperforms state-of-the-art methods in generating high-fidelity datasets while achieving superior cross-architecture generalization. Our work not only establishes a theoretical connection between diffusion priors and the objectives of dataset distillation but also provides a practical, training-free framework for improving the quality of the distilled dataset.", "builds_on": [ "minimax-2024", "d4m-2024", "mgd3-2025" ], "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "explained": { "before": "Minimax and MGD3 both add hand-designed representativeness/diversity objectives on top of a pretrained diffusion model -- Minimax by fine-tuning with minimax criteria, MGD3 by mode-discovery guidance -- treating representativeness as something that must be imposed from outside the generator.", "problem": "The paper argues these external constraints are unnecessary complexity: a well-trained diffusion model already has usable diversity and generalization priors (shown via low, matched train/test negative log-likelihood, Table 1) simply by virtue of its score-matching training objective and architecture- agnostic feature extraction, but no prior work exploits the model's own internal features as a representativeness signal, so they resort to bolt-on losses or clustering instead.", "idea": "Decompose the desired conditional score $\\nabla_x\\log p(x|\\mathcal{R})=\\nabla_x\\log p(x)+\\nabla_x\\log p(\\mathcal{R}|x)$: the first (unconditional) term already gives diversity+generalization for free from the pretrained model; the second (representativeness) term can be built, with no retraining, from a Mercer-kernel-induced distance between the diffusion backbone's own intermediate features on synthetic versus real samples, then injected as classifier-style energy guidance during sampling.", "method": "Treat the diffusion U-Net or DiT backbone as a feature map $\\phi:\\mathcal{X}\\to\\mathbb{R}^n$; define representativeness as a Boltzmann distribution over the average Mercer-kernel-induced distance $\\mathcal{D}_\\mathcal{K}$ between a synthetic sample's features and features of real training samples of the same class (linear kernel used in practice for tractability), proven (Theorems 3.1-3.2) to be a valid RKHS-norm distance so gradients are well-behaved. During VP-SDE reverse sampling, at each step the noisy synthetic latent and a noised real training sample of the target class are both passed through the same frozen backbone layer (the paper ablates which layer: U-Net \"Mid\" block for SD, early transformer blocks 4-12 for DiT); the gradient of the feature distance is added to the standard score/ noise prediction, scaled by guidance strength $\\gamma$ (Algorithm 1). No feature extractor, generator, or classifier is trained; the pretrained SD-v1.5 (general text-to-image) or DiT-XL/2-256 (ImageNet- pretrained, as used by Minimax) backbone is reused as both generator and its own representativeness critic.", "evidence": "ImageNet-1K, DiT backbone, ResNet-18 soft-label protocol (Table 2), IPC10/50: 49.1±1.2 / 62.7±1.5, beating IGD 45.5±0.5/59.8±0.3, MGD3 45.6±0.8/60.2±0.1, D3HR 44.3±0.3/59.4±0.1, VLCP 46.7±0.4/60.5±0.2 and Minimax 44.3±0.5/58.6±0.3 under one shared protocol. ImageNette/ImageWoof, hard-label protocol (Table 3), IPC10 ResNet-18: Nette 66.4±0.5 (IGD 67.7±0.3 slightly ahead, attributed by the authors to IGD's architecture-specific influence guidance baked in for ResNet-18); Woof IPC50 ResNetAP-10: 63.3±0.5 vs IGD 62.7±1.2, Minimax 59.8±0.8, MGD3 56.5±1.9. Isolating ablation (Fig. 5): feature-layer choice matters (mid U-Net layer / early DiT transformer blocks are best; final layers, which favor distribution alignment over representativeness, are worse) and increasing guidance scale $\\gamma$ raises representativeness (visualized as larger similarity \"sector area\" in Fig. 2 and higher accuracy) up to a point, after which it distorts the diversity/generalization gradient field and accuracy drops -- directly isolating the representativeness-guidance mechanism from the base model's inherent priors. t-SNE (Fig. 4) shows synthetic features matching real train/test manifolds with intra-class diversity preserved. Cost: training-free (no fine-tuning), so its added cost is only extra backbone forward/ backward passes for guidance at each sampling step; exact wall-clock not extracted from the main text.", "limitations": "Stated: future work needed to extend to newer generators (FLUX, SD3.5) and beyond vision. Observed: when applied to DiT, the backbone (DiT-XL/2-256) is itself pretrained on the full ImageNet target dataset, so DAP's strongest ImageNet-1K numbers still depend on a target-trained generator -- exactly the dependency CoDA later criticizes across the family; the guidance scale $\\gamma$ requires per- backbone/per-layer tuning (Fig. 5) and is reported to destabilize the other priors if set too high, so the training-free claim trades fine-tuning cost for a sampling-time hyperparameter search; the representativeness distance is computed per-class against real training samples at every guidance step, meaning (unlike D3HR/CoDA style one-shot statistics) it needs repeated access to real data features throughout sampling, not just at initialization." } }, { "id": "beyond-random-automatic-inner-2025", "type": "paper", "title": "Beyond Random: Automatic Inner-loop Optimization in Dataset Distillation", "arxiv": "2510.04838", "url": null, "date": "2025-10-06", "venue": { "name": "NeurIPS 2025", "status": "conference" }, "authors": [ "Muquan Li", "Hang Gou", "Dongyang Zhang", "Shuang Liang", "Xiurui Xie", "Deqiang Ouyang", "Ke Qin" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "meta-learning-bptt", "optimization-recipes" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Replaces the random truncation used in backpropagation-through-time dataset distillation with Automatic Truncated BPTT (AT-BPTT), which chooses truncation windows and sizes from stage-aware gradient statistics and a low-rank Hessian approximation, reporting a 6.16-point average accuracy gain over baselines on CIFAR-10/100, Tiny-ImageNet and ImageNet-1K along with a 3.9x speedup and 63% less memory.", "abstract": "The growing demand for efficient deep learning has positioned dataset distillation as a pivotal technique for compressing training dataset while preserving model performance. However, existing inner-loop optimization methods for dataset distillation typically rely on random truncation strategies, which lack flexibility and often yield suboptimal results. In this work, we observe that neural networks exhibit distinct learning dynamics across different training stages-early, middle, and late-making random truncation ineffective. To address this limitation, we propose Automatic Truncated Backpropagation Through Time (AT-BPTT), a novel framework that dynamically adapts both truncation positions and window sizes according to intrinsic gradient behavior. AT-BPTT introduces three key components: (1) a probabilistic mechanism for stage-aware timestep selection, (2) an adaptive window sizing strategy based on gradient variation, and (3) a low-rank Hessian approximation to reduce computational overhead. Extensive experiments on CIFAR-10, CIFAR-100, Tiny-ImageNet, and ImageNet-1K show that AT-BPTT achieves state-of-the-art performance, improving accuracy by an average of 6.16% over baseline methods. Moreover, our approach accelerates inner-loop optimization by 3.9x while saving 63% memory cost.", "awesome_sections": [ "Main / Better Optimization" ], "builds_on": [ "rat-bptt-2024" ] }, { "id": "deda-2025", "type": "paper", "title": "Diversity-Enhanced Distribution Alignment for Dataset Distillation", "short": "DEDA", "arxiv": null, "url": "https://openaccess.thecvf.com/content/ICCV2025/html/Li_Diversity-Enhanced_Distribution_Alignment_for_Dataset_Distillation_ICCV_2025_paper.html", "date": "2025-10", "venue": { "name": "ICCV 2025", "status": "conference" }, "authors": [ "Hongcheng Li", "Yucan Zhou", "Xiaoyan Gu", "Bo Li", "Weiping Wang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "cvf", "depth": "abstract" }, "summary": "Matches both class-wise means and covariance matrices of pretrained-model features between real and synthetic data in a Gaussian-distribution-alignment scheme, then adds a regularizer that maximizes diagonal and minimizes off-diagonal covariance terms in the last feature layer specifically, countering the low diversity and gradient starvation that mean/BN-statistics-only alignment produces; reports state-of-the-art results on CIFAR-10/100, Tiny-ImageNet and ImageNet-1K with no added compute.", "abstract": null, "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ], "builds_on": [ "dsdm-2024", "dm-2023" ] }, { "id": "gsdd-2025", "type": "paper", "title": "Parameterizing Dataset Distillation via Gaussian Splatting", "short": "GSDD", "arxiv": "2509.26219", "url": null, "date": "2025-09-30", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Chenyang Jiang", "Zhengcen Li", "Hang Zhao", "Qiben Shan", "Shaocong Wu", "Jingyong Su" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "gaussian" ] }, "tier": "catalogued", "links": { "code": "https://github.com/j-cyoung/GSDatasetDistillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Parameterizes each synthetic image as a set of 2D Gaussian splats rendered with CUDA-based batched splatting operators, giving a sparse yet expressive per-image representation that increases dataset diversity under a fixed storage budget; reports gains on standard benchmarks, ImageNet-1K, and video distillation.", "abstract": "Dataset distillation aims to compress training data while preserving training-aware knowledge, alleviating the reliance on large-scale datasets in modern model training. Dataset parameterization provides a more efficient storage structure for dataset distillation, reducing redundancy and accommodating richer information. However, existing methods either rely on complex auxiliary modules or fail to balance representational capacity and efficiency. In this paper, we propose GSDD, a simple, novel, and effective dataset parameterization technique for Dataset Distillation based on Gaussian Splatting. We adapt CUDA-based splatting operators for parallel training in batch, enabling high-quality rendering with minimal computational and memory overhead. Gaussian primitives can effectively capture meaningful training features, allowing a sparse yet expressive representation of individual images. Leveraging both high representational capacity and efficiency, GSDD substantially increases the diversity of distilled datasets under a given storage budget, thereby improving distillation performance. Beyond achieving competitive results on multiple standard benchmarks, GSDD also delivers significant performance gains on large-scale datasets such as ImageNet-1K and on video distillation tasks. In addition, we conduct comprehensive benchmarks to evaluate the computational efficiency, memory footprint, and cross-GPU architectural stability of GSDD. Code is available on https://github.com/j-cyoung/GSDatasetDistillation", "awesome_sections": [ "Main / Distilled Dataset Parametrization" ], "notes": "TITLE MISMATCH with arXiv: Parameterizing Dataset Distillation via Gaussian Splatting Draft's 'TITLE MISMATCH' note was a false positive: data/cache/arxiv.json confirms the same title, authors and date; treated as a normal verified arXiv draft." }, { "id": "high-order-progressive-trajectory-2025", "type": "paper", "title": "High-Order Progressive Trajectory Matching for Medical Image Dataset Distillation", "short": "HoP-TM", "arxiv": "2509.24177", "url": null, "date": "2025-09-29", "venue": { "name": "MICCAI 2025", "status": "conference" }, "authors": [ "Le Dong", "Jinghao Bian", "Jingyang Hou", "Jingliang Hu", "Yilei Shi", "Weisheng Dong", "Xiao Xiang Zhu", "Lichao Mou" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "trajectory-matching" ], "applications": [ "medical" ], "tier": "catalogued", "links": { "code": "https://github.com/Bian-jh/HoP-TM", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Extends trajectory matching for medical image distillation by matching a shape-wise potential that captures the geometry of intermediate points along expert parameter trajectories, not just terminal states, and by an easy-to-complex strategy that progressively matches parameters in order of complexity. Reports improved distillation performance on medical image classification while preserving privacy and keeping accuracy comparable to training on the original datasets.", "abstract": "Medical image analysis faces significant challenges in data sharing due to privacy regulations and complex institutional protocols. Dataset distillation offers a solution to address these challenges by synthesizing compact datasets that capture essential information from real, large medical datasets. Trajectory matching has emerged as a promising methodology for dataset distillation; however, existing methods primarily focus on terminal states, overlooking crucial information in intermediate optimization states. We address this limitation by proposing a shape-wise potential that captures the geometric structure of parameter trajectories, and an easy-to-complex matching strategy that progressively addresses parameters based on their complexity. Experiments on medical image classification tasks demonstrate that our method improves distillation performance while preserving privacy and maintaining model accuracy comparable to training on the original datasets. Our code is available at https://github.com/Bian-jh/HoP-TM.", "awesome_sections": [ "Applications / Medical" ], "notes": "" }, { "id": "rd3-2026", "type": "paper", "title": "Rectified Decoupled Dataset Distillation: A Closer Look for Fair and Comprehensive Evaluation", "short": "RD3", "arxiv": "2509.19743", "url": null, "date": "2025-09-24", "venue": { "name": "ICLR 2026", "status": "conference" }, "authors": [ "Xinhao Zhong", "Shuoyang Sun", "Xulin Gu", "Chenyang Zhu", "Bin Chen", "Yaowei Wang" ], "kind": "evaluation-benchmark", "settings": [ "image-classification" ], "families": [], "facets": { "scale": [ "imagenet-1k", "imagenet-subsets", "mnist-cifar", "tiny-imagenet" ] }, "tier": "notable", "links": { "code": "https://github.com/ndhg1213/RD3", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "Decoupled methods split into three sub-paradigms the paper explicitly separates for the first time: optimization-based (SRe2L, CDA, G-VBSM, DWA, EDC — pixel optimization against a pretrained classifier's cross-entropy and BN statistics), selection-based (RDED — crop/score/stitch real patches), and generation-based (D4M, Minimax — fine-tuned generative models or optimized visual-textual embeddings). Each paper's own post-evaluation recipe (batch size, LR schedule, epochs) differs; DC-BENCH (2022) and DD-Ranking (2025) had flagged augmentation and soft labels as confounds but had not isolated batch size/LR-decay-style protocol differences specifically within the decoupled family, nor quantified how much of the SRe2L-to-newer-method accuracy climb survives a fully unified protocol.", "problem": "On ImageNet-1K IPC10 with ResNet-18, the accuracy gap reported across published decoupled methods exceeds 27 percentage points; the paper's diagnosis is that this gap is inflated by inconsistent post-evaluation settings (batch size, LR decay, and other hyperparameters), not by differences in the intrinsic quality of the synthetic images.", "idea": "Re-run every representative decoupled method (across all three sub-paradigms) under one unified, standardized post-evaluation protocol and measure how much of the reported inter-method gap survives; separately, test whether a random-image-plus-soft-label baseline (in the spirit of label-worth-2024) still beats the \"state of the art\" once evaluation is unified.", "method": "No new synthesis method (despite the RD3 name, the contribution is the standardized evaluation framework, not a new distillation algorithm). Re-implements/re-evaluates SRe2L, CDA, G-VBSM, DWA, EDC (optimization-based), RDED (selection-based), D4M and Minimax (generation-based) under a single fixed post-evaluation recipe across CIFAR-10/100, TinyImageNet, ImageNette, ImageWoof and ImageNet-1K at multiple IPCs; also cross-evaluates on ResNet-18/50/101, MobileNet-V2, EfficientNet-B0, Swin-V2-T and ViT-B-16, and measures per-method synthesis wall-clock time (IPC=100, single RTX-3090) alongside accuracy. Separately reruns every method with hard labels and with a random-real-image-plus-soft-label control under the same unified protocol.", "evidence": "ImageNet-1K, ResNet-18, IPC10, relabel+KD regime (Fig. 1 / Table 2): before unification, published numbers span a 27.3-point range across methods; after RD3's unified protocol, the gap shrinks to 6.7 points, and every method's own accuracy rises substantially once evaluated under RD3's setting (Table 2 deltas: SRe2L +8.4, CDA +3.2, G-VBSM +3.9, DWA +4.0, EDC +2.1, RDED +2.4, D4M +5.0 at IPC50 — i.e. the *original* papers were themselves under-tuned relative to what a good but fixed protocol can achieve). Cross-architecture generalization at IPC50 (Table 3, single ResNet-18 teacher, relabel+KD): performance variance stays within ~5% across the ResNet family but diverges sharply on MobileNet-V2 and ViT-B-16, where SRe2L/CDA/DWA degrade substantially (attributed to low image diversity) while D4M (generation-based, high diversity) reaches SOTA across most architectures. Efficiency (Fig. 3, IPC100, single RTX-3090, excluding classifier-training time): synthesis time varies by up to 100x across methods despite near-identical accuracy — the paper argues efficiency, not accuracy, is now the more informative axis of comparison. Overlooked baseline (Section 5, Table 7): under the soft-label (relabel+KD) regime, simple random real-image sampling *beats every tested decoupled method* on CIFAR-10/100, ImageNette and ImageNet-1K (coarse-grained datasets, where random sampling maximizes diversity while staying consistent with the teacher's knowledge), though decoupled methods still win on fine-grained datasets (TinyImageNet, ImageWoof) where representative image selection matters more. Under hard labels, only Minimax and RDED (both producing realistic, class-representative images) beat random sampling across most datasets/ratios; optimization-based methods and D4M, which rely on soft-label alignment rather than hard-label-correlated images, fall far below random sampling with hard labels.", "limitations": "Confirms rather than resolves the underlying tension: the paper shows evaluation protocol explains most of the *inter-method* gap, but does not fully explain why *any* decoupled method beats hard- label random sampling in the first place (it attributes this to soft-label/teacher effects, echoing label-worth-2024 and DD-Ranking without adding a new mechanism). Efficiency comparisons explicitly exclude classifier/teacher training time, understating optimization- and selection-based methods' true deployment cost. Only covers the decoupled-synthesis paradigm (not gradient/trajectory/ distribution matching or coreset selection as first-class comparators, only as an appendix hard-label check). Single-GPU (RTX-3090) timing may not reflect cost at other hardware scales. The \"unified protocol\" is itself one specific choice of batch size/LR schedule among many plausible ones; the paper does not claim this specific choice is optimal, only that fixing *some* consistent choice collapses most of the reported gap." }, "summary": "RD3 systematically studies how post-evaluation choices (augmentation, epoch-wise soft labels, and other protocol details introduced after decoupled distillation methods such as SRe2L) affect reported test accuracy, and finds that much of the performance variation between existing decoupled-distillation methods reflects inconsistent evaluation protocols rather than differences in the intrinsic quality of the synthetic data. The paper proposes a standardized benchmark and evaluation protocol, and identifies general strategies that improve distilled-dataset effectiveness across settings.", "abstract": "Dataset distillation aims to generate compact synthetic datasets that enable models trained on them to achieve performance comparable to those trained on full real datasets, while substantially reducing storage and computational costs. Early bi-level optimization methods (e.g., MTT) have shown promising results on small-scale datasets, but their scalability is limited by high computational overhead. To address this limitation, recent decoupled dataset distillation methods (e.g., SRe$^2$L) separate the teacher model pre-training from the synthetic data generation process. These methods also introduce random data augmentation and epoch-wise soft labels during the post-evaluation phase to improve performance and generalization. However, existing decoupled distillation methods suffer from inconsistent post-evaluation protocols, which hinders progress in the field. In this work, we propose Rectified Decoupled Dataset Distillation (RD$^3$), and systematically investigate how different post-evaluation settings affect test accuracy. We further examine whether the reported performance differences across existing methods reflect true methodological advances or stem from discrepancies in evaluation procedures. Our analysis reveals that much of the performance variation can be attributed to inconsistent evaluation rather than differences in the intrinsic quality of the synthetic data. In addition, we identify general strategies that improve the effectiveness of distilled datasets across settings. By establishing a standardized benchmark and rigorous evaluation protocol, RD$^3$ provides a foundation for fair and reproducible comparisons in future dataset distillation research.", "notes": "Families left empty: this is an evaluation/protocol paper about the decoupled-synthesis family, not a new method within it." }, { "id": "edge-2025", "type": "paper", "title": "Efficient Multimodal Dataset Distillation via Generative Models", "short": "EDGE", "arxiv": "2509.15472", "url": null, "date": "2025-09-18", "venue": { "name": "NeurIPS 2025", "status": "conference" }, "authors": [ "Zhenghao Zhao", "Haoxuan Wang", "Junyi Wu", "Yuzhang Shang", "Gaowen Liu", "Yan Yan" ], "kind": "method", "settings": [ "vision-language" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "labels": "hard", "prior": [ "text-to-image-diffusion", "vision-language-model" ], "regime": "generator-fine-tuning", "parameterization": [ "pixels", "embedding-features" ], "scale": [ "flickr30k-coco" ] }, "tier": "core", "links": { "code": "https://github.com/ichbill/EDGE", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Proposes EDGE, which fine-tunes a pretrained Stable Diffusion v1.5 model for image-text dataset distillation by replacing its denoising loss with a bidirectional contrastive loss that aligns generated image and text embeddings and a minimax diversity loss that pushes different pairs' embeddings apart, then generates synthetic pairs by sampling from the fine-tuned model; evaluated with a pretrained NFNet image encoder and a frozen BERT-base text encoder, it reports on Flickr30K at 500 pairs 21.0/30.5% image-retrieval Recall@5/10 and 35.6/47.5% text-retrieval Recall@5/10 (Table 1), running about 18x faster than the trajectory-matching baseline LoRS.", "abstract": "Dataset distillation aims to synthesize a small dataset from a large dataset, enabling the model trained on it to perform well on the original dataset. With the blooming of large language models and multimodal large language models, the importance of multimodal datasets, particularly image-text datasets, has grown significantly. However, existing multimodal dataset distillation methods are constrained by the Matching Training Trajectories algorithm, which significantly increases the computing resource requirement, and takes days to process the distillation. In this work, we introduce EDGE, a generative distillation method for efficient multimodal dataset distillation. Specifically, we identify two key challenges of distilling multimodal datasets with generative models: 1) The lack of correlation between generated images and captions. 2) The lack of diversity among generated samples. To address the aforementioned issues, we propose a novel generative model training workflow with a bi-directional contrastive loss and a diversity loss. Furthermore, we propose a caption synthesis strategy to further improve text-to-image retrieval performance by introducing more text information. Our method is evaluated on Flickr30K, COCO, and CC3M datasets, demonstrating superior performance and efficiency compared to existing approaches. Notably, our method achieves results 18x faster than the state-of-the-art method.", "builds_on": [ "mtt-vl-2024", "lors-2024" ], "awesome_sections": [ "Main / Multimodal Distillation" ], "explained": { "before": "MTT-VL and LoRS both distill vision-language pairs by matching or unrolling expert training trajectories, which the paper documents as extremely costly at this setting's scale: MTT-VL needs about 4 days and up to 320GB memory for 100 pairs on Flickr30K, and LoRS needs about 150 GPU-hours for 500 pairs, because generating expert trajectories and unrolling the bi-level optimization both scale with the number of pairs.", "problem": "The paper argues generative distillation is the obvious way to cut this cost but identifies two failure modes specific to applying an off-the-shelf diffusion model to image-text data: (1) a diffusion model trained with a per-sample denoising objective has no incentive to make its generated images correspond to their conditioning captions, so image-text correlation - the thing image-text contrastive (ITC) training needs - is not guaranteed; and (2) with only a few hundred to a thousand pairs standing in for a dataset, generated-sample diversity is essential but not encouraged by the denoising loss either.", "idea": "Fine-tune a pretrained text-to-image diffusion model with two auxiliary losses in place of (part of) its denoising objective - a bidirectional image-text contrastive loss that pulls generated images and their conditioning captions together in embedding space while pushing apart mismatched pairs, and a minimax diversity loss that pushes generated samples' embeddings apart - then sample synthetic pairs once from the fine-tuned model, avoiding trajectory matching entirely.", "method": "A pretrained Stable Diffusion v1.5 is fine-tuned (a small subset of its parameters, following the Minimax-Diffusion recipe) on real captions sampled from the target dataset used as generation conditions. The bidirectional contrastive loss (Eqs. 3-4) is a cross-entropy over image-to-text and text-to-image similarity matrices computed from a batch of generated images and their conditioning captions' embeddings, combined as $\\mathcal L_C=\\lambda\\mathcal L_{I\\to T}+\\mathcal L_{T\\to I}$. The diversity (minimax) loss (Eq. 5) is the mean pairwise cosine similarity between normalized concatenated image-text embeddings across the batch, minimized to spread samples apart: $\\mathcal L_D=\\frac{2}{N(N-1)}\\sum_{i3$) reintroduces redundant local detail at the expense of global structure. Multi-model FADRM+ costs more than twice the time and memory of the single-model variant for its accuracy gain." }, "builds_on": [ "rded-2024", "edc-2024", "cv-dd-2025" ], "summary": "Introduces data-level residual (skip) connections -- Data Residual Matching -- to counter information vanishing during pixel-space synthesis, balancing newly optimized signal against core information already present in the raw data, while optimization-level refinements cut training time and peak GPU memory by half; reports 47.7% (single-model) and 50.0% (multi-model) top-1 on ImageNet-1K with ResNet-18 at 0.8% compression, ahead of RDED by 5.7 points and of EDC/CV-DD by 1.4/4.0 points.", "abstract": "Residual connection has been extensively studied and widely applied at the model architecture level. However, its potential in the more challenging data-centric approaches remains unexplored. In this work, we introduce the concept of Data Residual Matching for the first time, leveraging data-level skip connections to facilitate data generation and mitigate data information vanishing. This approach maintains a balance between newly acquired knowledge through pixel space optimization and existing core local information identification within raw data modalities, specifically for the dataset distillation task. Furthermore, by incorporating optimization-level refinements, our method significantly improves computational efficiency, achieving superior performance while reducing training time and peak GPU memory usage by 50%. Consequently, the proposed method Fast and Accurate Data Residual Matching for Dataset Distillation (FADRM) establishes a new state-of-the-art, demonstrating substantial improvements over existing methods across multiple dataset benchmarks in both efficiency and effectiveness. For instance, with ResNet-18 as the student model and a 0.8% compression ratio on ImageNet-1K, the method achieves 47.7% test accuracy in single-model dataset distillation and 50.0% in multi-model dataset distillation, surpassing RDED by +5.7% and outperforming state-of-the-art multi-model approaches, EDC and CV-DD, by +1.4% and +4.0%. Code is available at: https://github.com/Jiacheng8/FADRM.", "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "cao2-2025", "type": "paper", "title": "CaO$_2$: Rectifying Inconsistencies in Diffusion-Based Dataset Distillation", "short": "CaO2", "arxiv": "2506.22637", "url": null, "date": "2025-06-27", "venue": { "name": "ICCV 2025", "status": "conference" }, "authors": [ "Haoxuan Wang", "Zhenghao Zhao", "Junyi Wu", "Yuzhang Shang", "Gaowen Liu", "Yan Yan" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "labels": "mixed", "prior": [ "diffusion" ], "regime": "training-free", "scale": [ "imagenet-1k" ] }, "tier": "notable", "links": { "code": "https://github.com/hatchetProject/CaO2", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Identifies objective inconsistency (the distillation process diverges from the evaluation objective) and condition inconsistency (mismatches between generated images and their conditions) in diffusion-based distillation, and fixes both with a two-stage Condition-aware Optimization with Objective-guided Sampling: probability-informed sample selection, then latent refinement for higher conditional likelihood; reports a 2.3% average accuracy gain over the best baselines on ImageNet and its subsets.", "abstract": "The recent introduction of diffusion models in dataset distillation has shown promising potential in creating compact surrogate datasets for large, high-resolution target datasets, offering improved efficiency and performance over traditional bi-level/uni-level optimization methods. However, current diffusion-based dataset distillation approaches overlook the evaluation process and exhibit two critical inconsistencies in the distillation process: (1) Objective Inconsistency, where the distillation process diverges from the evaluation objective, and (2) Condition Inconsistency, leading to mismatches between generated images and their corresponding conditions. To resolve these issues, we introduce Condition-aware Optimization with Objective-guided Sampling (CaO$_2$), a two-stage diffusion-based framework that aligns the distillation process with the evaluation objective. The first stage employs a probability-informed sample selection pipeline, while the second stage refines the corresponding latent representations to improve conditional likelihood. CaO$_2$ achieves state-of-the-art performance on ImageNet and its subsets, surpassing the best-performing baselines by an average of 2.3% accuracy.", "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "explained": { "before": "Minimax Diffusion fine-tunes a diffusion model for representativeness/diversity, and D4M clusters latents into category prototypes; both generate images purely to match the target distribution, with no reference to how a classifier will actually use them. RDED instead selects and stitches real image crops using an evaluation-style pipeline. Matching-based methods (DC/DM/MTT/TESLA/DATM) do align distillation with the classification objective by construction, but do not scale efficiently to ImageNet.", "problem": "The paper names two specific failure modes never previously isolated: (1) Objective Inconsistency — a conditional diffusion model's images are optimized purely for generative fidelity/likelihood, so some generated images, when passed through a classifier, are predicted to belong to a different class than the one they were conditioned on; (2) Condition Inconsistency — because no real diffusion model achieves perfect conditional likelihood (only an ELBO is optimized, with nonzero training loss), a generated image's latent is never *exclusively* associated with its conditioning label, weakening the image-label correlation that supervised training on the distilled set depends on.", "idea": "Treat evaluation-time correctness as a two-stage post-hoc filter and refinement applied to an already-pretrained, untouched diffusion model: first over-generate a pool of candidate images per class and keep only the ones a lightweight external classifier confidently assigns to the correct class (fixing Objective Inconsistency), then nudge each kept image's latent, via a short diffusion-loss- minimizing optimization under its class condition, toward a region where the pretrained model's own conditional likelihood is higher (fixing Condition Inconsistency) — all without ever fine-tuning the generator.", "method": "A pretrained DiT (256x256, ImageNet-trained, untouched/frozen) is sampled (50 DDIM-style steps, fixed seed) to produce an image pool of size $mN$ per class ($m{=}2$ or $4$, $N{=}$IPC). Stage 1 (Objective- guided Sample Selection): a lightweight pretrained classifier (ResNet-18) scores the pool via softmax, and only images predicted as the conditioned class are kept, choosing the top-$N$ most confident (easy tasks/higher IPC per the task-oriented variation, Sec. 3.4) or least confident (harder tasks/lower IPC) correctly-classified samples; classes with insufficient correct samples backfill with random pool images. Stage 2 (Condition-aware Latent Optimization): each selected image's latent $\\mathbf{x}$ is perturbed by Gaussian noise to a randomly sampled early timestep $t\\in[1,\\hat T]$, $\\hat T \\ll T$, and optimized to minimize the diffusion denoising loss $\\mathbb{E}_{t,\\varepsilon}[\\|\\epsilon_\\theta(\\mathbf{x}_t,\\hat{\\mathbf{c}},t)-\\varepsilon\\|_2^2+ \\lambda\\|\\epsilon_\\theta(\\mathbf{x}_t,\\hat{\\mathbf{c}},t)-\\varepsilon\\|_\\infty]$ (Eq. 4, $L_\\infty$ regularization to bound the latent shift; 100 iterations, Adam, lr 0.0006, $\\lambda{=}10$), i.e. moving the latent toward a region of higher conditional likelihood under the frozen diffusion model itself, not toward a classifier's decision boundary. The condition $\\hat{\\mathbf{c}}$ used for this optimization is the true class label for easier tasks and an unconditional (classifier-free-guidance null) embedding for harder tasks (Eq. 5), chosen per-dataset from validation accuracy. Both stages generalize to a Masked Autoregressive (MAR) generation backbone and can be layered on top of an already Minimax-fine- tuned DiT as a plug-in.", "evidence": "ImageWoof/ImageNette, best-of-hard-or-soft-label protocol (paper reports whichever evaluation regime scores higher for each method, per method, per cell), ResNet-18/50/101 (Table 1): ImageNette IPC=10 ResNet-18 CaO2 65.0% vs. RDED 61.4%/Minimax 61.4% vs. SRe2L 29.4%; ImageWoof IPC=10 ResNet-18 CaO2 45.6% vs. RDED 38.5%/Minimax 40.1%. ImageNet-100/ImageNet-1K (Table 2, same mixed-best-protocol convention): ImageNet-1K IPC=10 ResNet-18 CaO2 46.1% vs. RDED 42.0% vs. Minimax 44.3% vs. D4M 27.9%; IPC=50 CaO2 60.0% vs. Minimax 58.6% vs. RDED 56.5%. Reported average gains over best baselines: +1.6% (ImageWoof), +4.3% (ImageNette), +1.8% (ImageNet-100), +1.5% (ImageNet-1K) — the abstract's headline \"2.3% average.\" Key ablation (Table 3, ImageWoof/ImageNette, ResNet-18): OSS (sample selection) alone and CLO (latent optimization) alone each give comparable, roughly equal-sized gains over the no-component baseline (e.g. ImageWoof IPC=10: baseline 38.7%, +OSS 42.6%, +CLO 42.1%, both combined 45.6%) — the paper reads this as evidence the two inconsistencies are separate, similarly-sized problems, not one problem in two guises. Regularization ablation (Table 4): $L_\\infty$ norm beats $L_1$, $L_2$, and no regularization. Selection-pool-size ablation (Fig. 3b): accuracy peaks at pool size 2x-4x IPC and does *not* keep improving with a larger pool, because a larger pool, while more diverse, can be less representative once filtered. Backbone generalization (Table 5): CaO2 improves a MAR backbone by +3.3%/+2.0% (ImageWoof/ImageNette averages) and, layered on top of an already Minimax-fine-tuned DiT, improves it further by +4.3%/+2.0% — evidence the method is complementary to, not merely a substitute for, fine- tuning-based diversity/representativeness objectives.", "limitations": "The DiT backbone is ImageNet-trained, i.e. matched to the target distribution for every reported number; the paper's own Limitations appendix (Sec. 11) states this explicitly as a constraint — \"DiTs can only deal with ImageNet classes\" — and notes that switching to a general text-to-image model (e.g. Stable Diffusion) to handle arbitrary classes is not tested here and would trade this limitation for larger model size and a loss of classification-aware constraints; the paper's own framing acknowledges the two inconsistencies stem from \"the fundamental difference between generation and discrimination,\" which its post-hoc filter-and-nudge approach only partially closes rather than resolving with a unified objective; the headline comparison table reports, per method, whichever of two different label-regime evaluation protocols (hard-label vs. soft-label via KD) scores higher, which makes Table 1/2 numbers not a single controlled label-regime comparison across methods, even though it is applied evenhandedly to CaO2 and all baselines alike." } }, { "id": "fedwsidd-2025", "type": "paper", "title": "FedWSIDD: Federated Whole Slide Image Classification via Dataset Distillation", "short": "FedWSIDD", "arxiv": "2506.15365", "url": null, "date": "2025-06-18", "venue": { "name": "MICCAI 2025", "status": "conference" }, "authors": [ "Haolong Jin", "Shenglin Liu", "Cong Cong", "Qingmin Feng", "Yongzhi Liu", "Lina Huang", "Yingzi Hu" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "applications": [ "federated-learning", "medical" ], "tier": "catalogued", "links": { "code": "https://github.com/f1oNae/FedWSIDD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Each client distills its whole-slide images into a compact set of synthetic slides, using a distillation algorithm that adds stain normalization for histopathology, and transmits the synthetic slides (not model parameters) to a server that aggregates and redistributes them; clients then train locally on synthetic plus real slides. Reports improved local WSI classification, tolerance of heterogeneous local models, and preserved patient privacy on CAMELYON16/17.", "abstract": "Federated learning (FL) has emerged as a promising approach for collaborative medical image analysis, enabling multiple institutions to build robust predictive models while preserving sensitive patient data. In the context of Whole Slide Image (WSI) classification, FL faces significant challenges, including heterogeneous computational resources across participating medical institutes and privacy concerns. To address these challenges, we propose FedWSIDD, a novel FL paradigm that leverages dataset distillation (DD) to learn and transmit synthetic slides. On the server side, FedWSIDD aggregates synthetic slides from participating centres and distributes them across all centres. On the client side, we introduce a novel DD algorithm tailored to histopathology datasets which incorporates stain normalisation into the distillation process to generate a compact set of highly informative synthetic slides. These synthetic slides, rather than model parameters, are transmitted to the server. After communication, the received synthetic slides are combined with original slides for local tasks. Extensive experiments on multiple WSI classification tasks, including CAMELYON16 and CAMELYON17, demonstrate that FedWSIDD offers flexibility for heterogeneous local models, enhances local WSI classification performance, and preserves patient privacy. This makes it a highly effective solution for complex WSI classification tasks. The code is available at FedWSIDD.", "awesome_sections": [ "Applications / Medical" ], "notes": "" }, { "id": "soft-label-leakage-2026", "type": "paper", "title": "Dataset distillation for memorized data: Soft labels can leak held-out teacher knowledge", "short": "Soft labels leak", "arxiv": "2506.14457", "url": null, "date": "2025-06-17", "venue": { "name": "ICLR 2026", "status": "conference" }, "authors": [ "Freya Behrens", "Lenka Zdeborová" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "label-distillation" ], "tier": "catalogued", "links": { "code": null, "project": null }, "facets": { "labels": "soft-static", "regime": "decoupled", "prior": [ "trained-experts" ] }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "Classic knowledge distillation (Hinton et al. 2015) and label-worth-2024 both treat soft labels as carriers of \"structured\" distributional information (inter-class similarity, dark knowledge) that helps a student generalize; theoretical accounts (Menon et al.'s statistical perspective on distillation, Boix-Adsera's theory of model distillation) likewise assume the teacher reflects meaningful structure in the data. Separately, the memorization literature (Zhang et al.'s random- label fitting result, Nanda et al.'s grokking/modular-addition setup) shows neural networks readily memorize facts unrelated to any generalizable structure. No prior work had asked whether soft-label distillation transfers a teacher's memorized, non-generalizing knowledge about specific training examples, as opposed to only generalizable structure.", "problem": "If a teacher used to relabel a distilled dataset has memorized specific facts about its training data (as large modern classifiers do), it is unknown whether a student trained only on that teacher's soft labels for a held-in fraction of examples can recover non-trivial information about the teacher's held-out memorized examples — a privacy/leakage question that is invisible in standard accuracy benchmarks, which cannot distinguish \"the student learned real structure\" from \"the student learned what the teacher happened to memorize.\"", "idea": "Isolate memorization from generalization by constructing settings where generalization is a priori impossible (finite i.i.d. random inputs with i.i.d. random labels), so that a teacher which fits the data must be purely memorizing it; then show a student trained on the teacher's soft labels for only a fraction of that data achieves non-trivial, sometimes perfect, accuracy on the held-out remainder it never observed. Softmax temperature acts as a dial between two student solutions: at low temperature the student recovers only class identity (soft labels approach one-hot, no leakage); at high temperature the student can recover the teacher's function almost exactly, including on the memorized held-out data.", "method": "Four settings, each partitioning a finite teacher training set $\\mathcal{D}^{\\text{T}}_\\star$ into a student-visible fraction $\\rho$ and a held-out remainder, with the student trained by cross-entropy against the teacher's temperature-$\\tau$ softmax soft labels $\\hat{\\mathbf{y}}=\\sigma_\\tau(f^\\star(\\mathbf{x}))$ only on the visible fraction: (1) a single-layer transformer on modular addition ($p{=}113$, the grokking/delayed-generalization setup of Nanda et al.), with teachers checkpointed at different points to obtain memorizing-only, partially- and fully-generalizing teachers; (2) multinomial logistic regression on i.i.d. Gaussian inputs with i.i.d. random labels, where three sample-complexity thresholds $\\alpha=n/d$ are derived in closed form in the high-dimensional proportional limit — teacher memorization capacity $\\alpha^{\\text{T}}_{\\text{label}}\\le 2$ (Cover's theorem), student identifiability $\\alpha^{\\text{S}}_{\\text{id}}(\\rho)=1/\\rho$, and student memorization capacity $\\alpha^{\\text{S}}_{\\text{label}}(\\rho)$; (3) matched-capacity single-hidden-layer ReLU MLPs (multi-class, up to $c{=}100$), where two distinct leaking solutions are identified — one that memorizes the soft-label vectors themselves, another that functionally matches the teacher — with a sharp phase transition between them at the identifiability threshold; (4) a pretrained GPT-2 with a linear classification head fine-tuned to memorize 6000 random three-number token sequences mapped to random classes, to test whether the phenomenon persists in a large pretrained architecture. In the loop: one pretrained/fitted teacher per setting (frozen once fit); no image synthesis or bi-level optimization — the \"distilled\" artifact is purely the teacher's per-sample soft-label vector.", "evidence": "Modular addition, single-layer transformer, $p{=}113$, $\\tau{=}10$ (Fig. 2): for a non-generalizing (memorizing-only) teacher, the student's accuracy on the held-out memorized set $\\mathcal{D}^{\\text{S}}_{\\text{test}}$ reaches 1.0 as $\\rho$ grows, while its accuracy on true task-generalization data $\\mathcal{D}_{\\text{val}}$ stays near the (low) teacher's own validation accuracy — the leaked information is specific to what the teacher memorized, not the underlying modular-addition structure. Binary logistic regression, $d{=}1600$, $\\rho{=}0.8$ (Fig. 3): held-out test accuracy on memorized data grows monotonically with $\\alpha{=}n/d$ and reaches $\\ge$99% at $\\alpha^{\\text{S}}_{\\text{id}}(\\rho{=}0.8)\\simeq1.26$ — a fifth of the \"memorized\" data is recovered essentially perfectly from soft labels alone, just above the identifiability threshold. GPT-2 on random sequences (Sec. 5.3, Fig. 7): teacher fits 6000 memorized sequence-class pairs (1000 possible classes, random-guess baseline $\\approx$0.1%); students trained on soft labels for $\\rho=\\{0.2,0.5,0.8\\}$ of that set reach held-out test accuracy $\\{21.3\\%,52.4\\%,65.2\\%\\}$ while training accuracy is $\\approx$99.5% in all three cases — at $\\rho{=}0.8$ the student recovers >60% accuracy on sequences it never saw, from a teacher whose only \"knowledge\" is memorization by construction. Multi-class scaling (Fig. 4B, $\\rho{=}0.8$, $\\tau{=}10$): the four relevant capacity/ leakage thresholds scale differently with class count $c$ ($\\alpha^{\\text{S}}_{\\text{id}}\\sim1/c$; $\\alpha^{\\text{T}}_{\\text{label}}\\sim\\alpha^{\\text{S}}_{\\text{label}}\\sim1/\\log c$; a class-shuffled control threshold $\\alpha^{\\text{S-shuffle}}_{\\text{label}}\\sim1/\\sqrt{c}$).", "limitations": "Explicitly restricted to synthetic, structured-or-explicitly-memorized data (modular addition, i.i.d. random Gaussian/label data, random token sequences), not natural data distributions; uses simple models (logistic regression, one-hidden-layer ReLU MLPs, small transformers) chosen for theoretical tractability, plus a single GPT-2 experiment whose leakage thresholds are not shown to generalize to deeper architectures; does not explore regularization or optimizer choices as factors in capacity; teachers are constructed to either purely memorize or purely generalize, not the realistic mixture large models exhibit; no privacy defense or mitigation is proposed or evaluated, left as future work (\"whether it can be hidden for privacy reasons\")." }, "summary": "Shows that students trained on a teacher's soft labels can achieve non-trivial, sometimes near-perfect, accuracy on held-out data the teacher memorized but never generalized to, including on i.i.d. random datasets where generalization is a priori impossible, and that with enough soft labels the student can recover the teacher's predictions on all inputs, including the memorized held-out ones; the effect depends strongly on the softmax temperature used for the labels but persists across network capacities, architectures and dataset compositions.", "abstract": "Dataset distillation aims to compress training data into fewer examples via a teacher, from which a student can learn effectively. While its success is often attributed to structure in the data, modern neural networks also memorize specific facts, but if and how such memorized information is can transferred in distillation settings remains less understood. In this work, we show that students trained on soft labels from teachers can achieve non-trivial accuracy on held-out memorized data they never directly observed. This effect persists on structured data when the teacher has not generalized.To analyze it in isolation, we consider finite random i.i.d. datasets where generalization is a priori impossible and a successful teacher fit implies pure memorization. Still, students can learn non-trivial information about the held-out data, in some cases up to perfect accuracy. In those settings, enough soft labels are available to recover the teacher functionally - the student matches the teacher's predictions on all possible inputs, including the held-out memorized data. We show that these phenomena strongly depend on the temperature with which the logits are smoothed, but persist across varying network capacities, architectures and dataset compositions.", "awesome_sections": [ "Main / Better Understanding" ], "applications": [ "privacy" ] }, { "id": "flowing-datasets-with-wasserstein-2025", "type": "paper", "title": "Flowing Datasets with Wasserstein over Wasserstein Gradient Flows", "arxiv": "2506.07534", "url": null, "date": "2025-06-09", "venue": { "name": "ICML 2025", "status": "conference" }, "authors": [ "Clément Bonet", "Christophe Vauthier", "Anna Korba" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [], "tier": "catalogued", "links": { "code": "https://github.com/clbonet/Flowing_Datasets_with_WoW_Gradient_Flows", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Represents a labeled dataset as a mixture distribution over per-class feature distributions and defines gradient flows on this space of distributions-over-distributions using a Wasserstein-over-Wasserstein metric, applying the resulting dynamics with a sliced-Wasserstein-kernel maximum-mean-discrepancy objective to transfer-learning and dataset-distillation tasks.", "abstract": "Many applications in machine learning involve data represented as probability distributions. The emergence of such data requires radically novel techniques to design tractable gradient flows on probability distributions over this type of (infinite-dimensional) objects. For instance, being able to flow labeled datasets is a core task for applications ranging from domain adaptation to transfer learning or dataset distillation. In this setting, we propose to represent each class by the associated conditional distribution of features, and to model the dataset as a mixture distribution supported on these classes (which are themselves probability distributions), meaning that labeled datasets can be seen as probability distributions over probability distributions. We endow this space with a metric structure from optimal transport, namely the Wasserstein over Wasserstein (WoW) distance, derive a differential structure on this space, and define WoW gradient flows. The latter enables to design dynamics over this space that decrease a given objective functional. We apply our framework to transfer learning and dataset distillation tasks, leveraging our gradient flow construction as well as novel tractable functionals that take the form of Maximum Mean Discrepancies with Sliced-Wasserstein based kernels between probability distributions.", "awesome_sections": [ "Main / Better Understanding" ] }, { "id": "od3-2026", "type": "paper", "title": "OD3: Optimization-free Dataset Distillation for Object Detection", "short": "OD3", "arxiv": "2506.01942", "url": null, "date": "2025-06-02", "venue": { "name": "ICLR 2026", "status": "conference" }, "authors": [ "Salwa K. Al Khatib", "Ahmed ElHagry", "Shitong Shao", "Zhiqiang Shen" ], "kind": "method", "settings": [ "dense-prediction" ], "families": [ "decoupled-synthesis", "coreset-selection" ], "builds_on": [ "dcod-2024", "rded-2024", "sre2l-2023" ], "tier": "notable", "links": { "code": "https://github.com/VILA-Lab/OD3", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "OD3 distills an object-detection dataset without any bi-level optimization, in two stages it iteratively places real object instances at suitable locations in synthesized images (candidate selection), then screens out low-confidence placements with a pretrained observer model (candidate screening). On MS COCO and PASCAL VOC at compression ratios from 0.25% to 5%, the paper reports new state-of-the-art results, surpassing the prior detection-distillation method by more than 14 points of mAP50 at a 1.0% compression ratio.", "abstract": "Training large neural networks on large-scale datasets requires substantial computational resources, particularly for dense prediction tasks such as object detection. Although dataset distillation (DD) has been proposed to alleviate these demands by synthesizing compact datasets from larger ones, most existing work focuses solely on image classification, leaving the more complex detection setting largely unexplored. In this paper, we introduce OD3, a novel optimization-free data distillation framework specifically designed for object detection. Our approach involves two stages: first, a candidate selection process in which object instances are iteratively placed in synthesized images based on their suitable locations, and second, a candidate screening process using a pre-trained observer model to remove low-confidence objects. We perform our data synthesis framework on MS COCO and PASCAL VOC, two popular detection datasets, with compression ratios ranging from 0.25% to 5%. Compared to the prior solely existing dataset distillation method on detection and conventional core set selection methods, OD3 delivers superior accuracy, establishes new state-of-the-art results, surpassing prior best method by more than 14% on COCO mAP50 at a compression ratio of 1.0%. Code is available at: https://github.com/VILA-Lab/OD3.", "notes": "", "explained": { "before": "The only prior detection-specific method, DCOD, still relies on uni-level optimization (model inversion of a trained detector, using pixel/feature regularization) to synthesize images, following SRe2L's inversion paradigm; coreset-selection methods for detection (CSOD, training-free instance- segmentation pruning) select whole real images or patches but typically only reach compression ratios above 20%, far coarser than what classification-style distillation achieves.", "problem": "Any optimization-based synthesis (bi-level as in classification DD, or DCOD's uni-level inversion) is costly and, in DCOD's case specifically, is tied to fixed gradients and pixel-specific updates that cannot flexibly add or remove context around objects of different sizes; small objects in particular lack sufficient contextual information once inverted, and no detection-condensation method reaches the very low compression ratios (sub-1%) that classification-style decoupled-synthesis methods do.", "idea": "Skip optimization entirely: build each synthetic \"canvas\" by iteratively copy-pasting real object crops (with their real bounding boxes and classes) onto a blank image at random non-overlapping positions (candidate selection), then run a pretrained \"observer\" detector on the assembled canvas and discard objects it does not confidently recognize (candidate screening) — an add-then-remove loop that a theorem shows strictly increases an \"information density x diversity\" objective over an add-only baseline.", "method": "For each of IPD (images-per-dataset) synthetic canvases, repeatedly sample bounding-box crops $\\langle b_{ir}, c_{ir}\\rangle$ from the real dataset, extend each crop's box by a Scale-Aware Dynamic Context Extension (SA-DCE) amount $\\ell_{extension}=(1-\\frac{a(o_{ir})-\\bar a_{min}}{\\bar a_{max}-\\bar a_{min}})\\times\\bar r$ (smaller objects get proportionally more added context), and place the crop at a random position if its IoU with existing canvas content is below overlap threshold $\\tau{=}0.6$ (up to $M{=}40$ placement attempts). Once a canvas is assembled, a pretrained observer detector $\\theta_{obs}$ (Faster R-CNN-101 for COCO) scores every placed object; objects below confidence threshold $\\eta{=}0.2$ are removed. This add-then-remove cycle (Eq. 9, $f_{remove}\\circ f_{add}$) is proven (Theorem 1, Appendix E) to reach an information-density-plus-diversity objective $G_2\\geq G_1$, i.e. at least as high as never removing anything. At post-evaluation time, a target detector is trained on the final canvases using feature-level soft labels: rather than logit-based KD (found ineffective for detection), a PKD-style (Pearson-correlation-based) channel-normalized FPN feature target is distilled from the observer into the target model's backbone+FPN (Eq. 5-6). Stored artifact: assembled canvas images with real bounding boxes/classes; the observer model's FPN features are used only as a training-time soft label, not stored as part of the compressed dataset itself.", "evidence": "Table 1 (MS COCO, Faster R-CNN-101 observer, Faster R-CNN-50 target/evaluator, 4 seeds ± SEM): at IPD 1.0%, OD3 reaches 22.40% mAP / 39.50% mAP50 / 22.90% mAP75 vs. DCOD 12.10% / 24.70% / 10.40% (full dataset 39.80% / 60.10% / 43.30%) — a +10.3 mAP / +14.8 mAP50 / +12.5 mAP75 gain over the prior detection- distillation SOTA, and roughly 5-6x the best coreset baseline (Random 8.30% mAP). At the lowest ratio tested, 0.25%: OD3 12.90% mAP vs. DCOD 7.20% (+5.7). Table 2 (Pascal VOC, Faster R-CNN-50 observer=target, mAP50): at IPD 2.0%, OD3 58.70% vs. DCOD 50.70% (+8.0) vs. full dataset 80.35%; the margin over DCOD grows with compression ratio (+0.6 at 0.5%, +4.7 at 1.0%, +8.0 at 2.0%). Table 3 (ablation, label type, COCO 0.25-1.0%): SA-DCE extended boxes (Ex-BBox) consistently beat plain ground-truth boxes (Bbox) and segmentation-mask labels (Mask) across nearly every metric (e.g. 0.5% IPD: 17.20% vs. 16.60% vs. 15.10% mAP), isolating the context-extension mechanism as a positive contributor though a modest one (+0.4-0.6 mAP over Bbox) relative to the overall gain over DCOD.", "limitations": "Relies on ground-truth bounding boxes during synthesis (the paper's own stated limitation), restricting applicability to fully unsupervised or label-scarce settings — the method redistributes real annotated crops rather than synthesizing genuinely novel appearance; the two overlap/confidence thresholds ($\\tau{=}0.6$, $\\eta{=}0.2$) and placement-attempt budget ($M{=}40$) are fixed hyperparameters set once rather than tuned per dataset; the observer model used for screening and soft-label generation must itself be a detector already trained on (or near) the target distribution, so the method is not fully label-free even though it needs no bi-level optimization; the theoretical guarantee (Theorem 1) only shows the add-then-remove objective dominates add-only, not that the chosen objective itself correlates tightly with downstream detector accuracy." } }, { "id": "optical-2025", "type": "paper", "title": "OPTICAL: Leveraging Optimal Transport for Contribution Allocation in Dataset Distillation", "short": "OPTICAL", "arxiv": null, "url": "https://openaccess.thecvf.com/content/CVPR2025/html/Cui_OPTICAL_Leveraging_Optimal_Transport_for_Contribution_Allocation_in_Dataset_Distillation_CVPR_2025_paper.html", "date": "2025-06", "venue": { "name": "CVPR 2025", "status": "conference" }, "authors": [ "Xiao Cui", "Yulei Qin", "Wengang Zhou", "Hongsheng Li", "Houqiang Li" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "cvf", "depth": "abstract" }, "summary": "Argues that sample-generation-based distillation methods give every real instance equal, uniform contribution when shaping each synthetic sample and so ignore instance-level real-synthetic relationships, and reformulates the matching objective as a bi-level matching-and-approximating problem where an optimal-transport matrix allocates contributions from real instances before the synthetic samples are refined against that allocation; reports gains across seven datasets and three architectures as a plug-in compatible with multiple distillation frameworks.", "abstract": null, "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ] }, { "id": "hdd-2025", "type": "paper", "title": "Hyperbolic Dataset Distillation", "short": "HDD", "arxiv": "2505.24623", "url": null, "date": "2025-05-30", "venue": { "name": "NeurIPS 2025", "status": "conference" }, "authors": [ "Wenyuan Li", "Guang Li", "Keisuke Maeda", "Takahiro Ogawa", "Miki Haseyama" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "tier": "notable", "links": { "code": "https://github.com/Guang000/HDD", "project": "https://guang000.github.io/HDD-Webpage/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Embeds pretrained-network features into Lorentz hyperbolic space and matches the hyperbolic (geodesic) distance between synthetic and real class centroids instead of Euclidean distribution matching, explicitly encoding the data's hierarchical structure into the distilled set; pruning in hyperbolic space is shown to retain model performance with only 20% of the distilled set.", "abstract": "To address the computational and storage challenges posed by large-scale datasets in deep learning, dataset distillation has been proposed to synthesize a compact dataset that replaces the original while maintaining comparable model performance. Unlike optimization-based approaches that require costly bi-level optimization, distribution matching (DM) methods improve efficiency by aligning the distributions of synthetic and original data, thereby eliminating nested optimization. DM achieves high computational efficiency and has emerged as a promising solution. However, existing DM methods, constrained to Euclidean space, treat data as independent and identically distributed points, overlooking complex geometric and hierarchical relationships. To overcome this limitation, we propose a novel hyperbolic dataset distillation method, termed HDD. Hyperbolic space, characterized by negative curvature and exponential volume growth with distance, naturally models hierarchical and tree-like structures. HDD embeds features extracted by a shallow network into the Lorentz hyperbolic space, where the discrepancy between synthetic and original data is measured by the hyperbolic (geodesic) distance between their centroids. By optimizing this distance, the hierarchical structure is explicitly integrated into the distillation process, guiding synthetic samples to gravitate towards the root-centric regions of the original data distribution while preserving their underlying geometric characteristics. Furthermore, we find that pruning in hyperbolic space requires only 20% of the distilled core set to retain model performance, while significantly improving training stability. To the best of our knowledge, this is the first work to incorporate the hyperbolic space into the dataset distillation process. The code is available at https://github.com/Guang000/HDD.", "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ], "builds_on": [ "dm-2023" ] }, { "id": "prism-2026-2", "type": "paper", "title": "PRISM: Video Dataset Condensation with Progressive Refinement and Insertion for Sparse Motion", "short": "PRISM", "arxiv": "2505.22564", "url": null, "date": "2025-05-28", "venue": { "name": "CVPR 2026", "status": "conference" }, "authors": [ "Jaehyun Choi", "Jiwan Hur", "Gyojin Han", "Jaemyung Yu", "Junmo Kim" ], "kind": "method", "settings": [ "video" ], "families": [ "gradient-matching", "optimization-recipes" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "PRISM treats a condensed video as one fully coupled spatiotemporal structure rather than disentangling it into separate static and dynamic components, starting from minimal temporal anchors and progressively inserting key-frames only where linear interpolation between them fails to capture non-linear motion, guided by gradient misalignments between synthetic and real data. The paper reports competitive accuracy with state-of-the-art storage efficiency on standard video-distillation benchmarks by allocating representational capacity only where the motion needs it.", "abstract": "Video dataset condensation aims to reduce the immense computational cost of video processing. However, it faces a fundamental challenge regarding the inseparable interdependence between spatial appearance and temporal dynamics. Prior work follows a static/dynamic disentanglement paradigm where videos are decomposed into static content and auxiliary motion signals. This multi-stage approach often misrepresents the intrinsic coupling of real-world actions. We introduce Progressive Refinement and Insertion for Sparse Motion (PRISM), a holistic approach that treats the video as a unified and fully coupled spatiotemporal structure from the outset. To maximize representational efficiency, PRISM addresses the inherent temporal redundancy of video by avoiding fixed-frame optimization. It begins with minimal temporal anchors and progressively inserts key-frames only where linear interpolation fails to capture non-linear dynamics. These critical moments are identified through gradient misalignments. Such an adaptive process ensures that representational capacity is allocated precisely where needed, minimizing storage requirements while preserving complex motion. Extensive experiments demonstrate that PRISM achieves competitive performance across standard benchmarks while providing state-of-the-art storage efficiency through its sparse and holistically learned representation.", "notes": "Family inferred from \"gradient misalignments\" as the matching signal; the abstract does not name the backbone matching objective explicitly, so gradient-matching is a best-effort guess (see handoff doubts)." }, { "id": "dynamic-aware-video-distillation-2025", "type": "paper", "title": "Dynamic-Aware Video Distillation: Optimizing Temporal Resolution Based on Video Semantics", "short": "DAViD", "arxiv": "2506.02021", "url": null, "date": "2025-05-28", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Yinjie Zhao", "Heng Zhao", "Bihan Wen", "Yew-Soon Ong", "Joey Tianyi Zhou" ], "kind": "method", "settings": [ "video" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Trains a reinforcement-learning agent, updated with a teacher-in-the-loop reward, to predict a per-class temporal resolution for synthetic video clips instead of assuming uniform temporal redundancy across classes, which the paper identifies as a limitation of prior video distillation methods. Reports substantial accuracy improvements over existing video dataset distillation methods.", "abstract": "With the rapid development of vision tasks and the scaling on datasets and models, redundancy reduction in vision datasets has become a key area of research. To address this issue, dataset distillation (DD) has emerged as a promising approach to generating highly compact synthetic datasets with significantly less redundancy while preserving essential information. However, while DD has been extensively studied for image datasets, DD on video datasets remains underexplored. Video datasets present unique challenges due to the presence of temporal information and varying levels of redundancy across different classes. Existing DD approaches assume a uniform level of temporal redundancy across all different video semantics, which limits their effectiveness on video datasets. In this work, we propose Dynamic-Aware Video Distillation (DAViD), a Reinforcement Learning (RL) approach to predict the optimal Temporal Resolution of the synthetic videos. A teacher-in-the-loop reward function is proposed to update the RL agent policy. To the best of our knowledge, this is the first study to introduce adaptive temporal resolution based on video semantics in video dataset distillation. Our approach significantly outperforms existing DD methods, demonstrating substantial improvements in performance. This work paves the way for future research on more efficient and semantic-adaptive video dataset distillation research.", "notes": "" }, { "id": "temporal-saliency-guided-distillation-2025", "type": "paper", "title": "Temporal Saliency-Guided Distillation: A Scalable Framework for Distilling Video Datasets", "arxiv": "2505.20694", "url": null, "date": "2025-05-27", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Xulin Gu", "Xinhao Zhong", "Zhixing Wei", "Yimin Zhou", "Shuoyang Sun", "Bin Chen", "Hongpeng Wang", "Yuan Luo" ], "kind": "method", "settings": [ "video" ], "families": [ "decoupled-synthesis", "optimization-recipes" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Optimizes synthetic video clips directly against a single pretrained model rather than through a bi-level inner training loop, and adds a temporal-saliency-guided filtering mechanism that uses inter-frame differences to concentrate distillation on informative temporal cues while suppressing redundant frames. Reports state-of-the-art results on standard video dataset-distillation benchmarks.", "abstract": "Dataset distillation (DD) has emerged as a powerful paradigm for dataset compression, enabling the synthesis of compact surrogate datasets that approximate the training utility of large-scale ones. While significant progress has been achieved in distilling image datasets, extending DD to the video domain remains challenging due to the high dimensionality and temporal complexity inherent in video data. Existing video distillation (VD) methods often suffer from excessive computational costs and struggle to preserve temporal dynamics, as na\\\"ive extensions of image-based approaches typically lead to degraded performance. In this paper, we propose a novel uni-level video dataset distillation framework that directly optimizes synthetic videos with respect to a pre-trained model. To address temporal redundancy and enhance motion preservation, we introduce a temporal saliency-guided filtering mechanism that leverages inter-frame differences to guide the distillation process, encouraging the retention of informative temporal cues while suppressing frame-level redundancy. Extensive experiments on standard video benchmarks demonstrate that our method achieves state-of-the-art performance, bridging the gap between real and distilled video data and offering a scalable solution for video dataset compression.", "notes": "Uni-level optimization against a fixed pretrained model most resembles decoupled-synthesis; the abstract does not fully specify the matching objective — worth a second look." }, { "id": "diversity-driven-generative-dataset-2025", "type": "paper", "title": "Diversity-Driven Generative Dataset Distillation Based on Diffusion Model with Self-Adaptive Memory", "arxiv": "2505.19469", "url": null, "date": "2025-05-26", "venue": { "name": "ICIP 2025", "status": "conference" }, "authors": [ "Mingzhuo Li", "Guang Li", "Jiafeng Mao", "Takahiro Ogawa", "Miki Haseyama" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Uses a self-adaptive memory to measure how closely a diffusion-generated distilled set's distribution aligns with the real dataset, and feeds that alignment signal back into sampling to steer the diffusion model toward generating more diverse images during distillation. Reports outperforming existing generative dataset-distillation methods in most settings.", "abstract": "Dataset distillation enables the training of deep neural networks with comparable performance in significantly reduced time by compressing large datasets into small and representative ones. Although the introduction of generative models has made great achievements in this field, the distributions of their distilled datasets are not diverse enough to represent the original ones, leading to a decrease in downstream validation accuracy. In this paper, we present a diversity-driven generative dataset distillation method based on a diffusion model to solve this problem. We introduce self-adaptive memory to align the distribution between distilled and real datasets, assessing the representativeness. The degree of alignment leads the diffusion model to generate more diverse datasets during the distillation process. Extensive experiments show that our method outperforms existing state-of-the-art methods in most situations, proving its ability to tackle dataset distillation tasks.", "notes": "" }, { "id": "data-distill-net-a-2025", "type": "paper", "title": "Data-Distill-Net: A Data Distillation Approach Tailored for Reply-based Continual Learning", "short": "Data-Distill-Net", "arxiv": "2505.20135", "url": null, "date": "2025-05-26", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Wenyang Liao", "Quanziang Wang", "Yichen Wu", "Renzhen Wang", "Deyu Meng" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "label-distillation" ], "applications": [ "continual-learning" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "For replay-based continual learning, keeps a learnable memory buffer of real replay images but, instead of optimizing the images themselves, trains a lightweight module that generates learned soft labels distilling both the current task's data and knowledge accumulated in the prior buffer. Reports competitive accuracy and reduced forgetting across continual-learning benchmarks relative to standard replay baselines, at lower overhead than distilling the buffer images directly.", "abstract": "Replay-based continual learning (CL) methods assume that models trained on a small subset can also effectively minimize the empirical risk of the complete dataset. These methods maintain a memory buffer that stores a sampled subset of data from previous tasks to consolidate past knowledge. However, this assumption is not guaranteed in practice due to the limited capacity of the memory buffer and the heuristic criteria used for buffer data selection. To address this issue, we propose a new dataset distillation framework tailored for CL, which maintains a learnable memory buffer to distill the global information from the current task data and accumulated knowledge preserved in the previous memory buffer. Moreover, to avoid the computational overhead and overfitting risks associated with parameterizing the entire buffer during distillation, we introduce a lightweight distillation module that can achieve global information distillation solely by generating learnable soft labels for the memory buffer data. Extensive experiments show that, our method can achieve competitive results and effectively mitigates forgetting across various datasets. The source code will be publicly available.", "notes": "" }, { "id": "mgd3-2025", "type": "paper", "title": "MGD$^3$: Mode-Guided Dataset Distillation using Diffusion Models", "short": "MGD³", "arxiv": "2505.18963", "url": null, "date": "2025-05-25", "venue": { "name": "ICML 2025", "status": "conference" }, "authors": [ "Jeffrey A. Chan-Santiago", "Praveen Tirupattur", "Gaurav Kumar Nayak", "Gaowen Liu", "Mubarak Shah" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "labels": "mixed", "prior": [ "diffusion", "text-to-image-diffusion" ], "regime": "training-free", "scale": [ "imagenet-1k" ] }, "tier": "notable", "links": { "code": "https://github.com/jachansantiago/mode_guidance/", "project": "https://jachansantiago.com/mode-guided-distillation/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Uses a pretrained diffusion model with no distillation-loss fine-tuning, addressing diversity in three stages: Mode Discovery finds distinct data modes, Mode Guidance steers sampling toward intra-class diversity, and Stop Guidance halts guidance early enough to avoid artifacts; reports accuracy gains of 4.4%, 2.9%, 1.6% and 1.6% on ImageNette, ImageIDC, ImageNet-100 and ImageNet-1K over the state of the art, at much lower computational cost than fine-tuning-based methods.", "abstract": "Dataset distillation has emerged as an effective strategy, significantly reducing training costs and facilitating more efficient model deployment. Recent advances have leveraged generative models to distill datasets by capturing the underlying data distribution. Unfortunately, existing methods require model fine-tuning with distillation losses to encourage diversity and representativeness. However, these methods do not guarantee sample diversity, limiting their performance. We propose a mode-guided diffusion model leveraging a pre-trained diffusion model without the need to fine-tune with distillation losses. Our approach addresses dataset diversity in three stages: Mode Discovery to identify distinct data modes, Mode Guidance to enhance intra-class diversity, and Stop Guidance to mitigate artifacts in synthetic samples that affect performance. Our approach outperforms state-of-the-art methods, achieving accuracy gains of 4.4%, 2.9%, 1.6%, and 1.6% on ImageNette, ImageIDC, ImageNet-100, and ImageNet-1K, respectively. Our method eliminates the need for fine-tuning diffusion models with distillation losses, significantly reducing computational costs. Our code is available on the project webpage: https://jachansantiago.github.io/mode-guided-distillation/", "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "explained": { "before": "D4M clusters real-image latents but samples from the noised modes directly, which the paper argues limits diversity; Minimax Diffusion fixes diversity by fine-tuning the diffusion model itself with minimax representativeness/diversity losses, at real computational cost (Minimax needs ~10 GPU-hours to fine-tune for ImageNet-100 IPC=10, and the paper reports IDC-1 needs >100). GLaD/H-GLaD/LD3M optimize generative latents with a matching objective but do not scale past small IPC/resolution (20 IPC at 256x256 is called out as their practical ceiling).", "problem": "Pretrained diffusion models oversample the densest modes of a class's data distribution, and when the number of dominant modes is smaller than the target IPC this produces redundant, near-duplicate samples — a diversity failure that persists even after Minimax's fine-tuning (the paper's t-SNE analysis, Fig. 5-6, shows Minimax reaches broader but still incomplete coverage) and that costs real GPU time to fine-tune around.", "idea": "Fix the diversity problem entirely at sampling time, without ever touching the diffusion model's weights: discover a class's distinct modes by clustering real-image latents, steer each sample's denoising trajectory toward a specific assigned mode with a lightweight guidance term, and stop that guidance before the final refinement stage of denoising so the model can still finish each sample with unconstrained, artifact-free detail.", "method": "A frozen DiT-XL/2 (or, in a robustness check, LDM's U-Net, or a general-purpose text-to-image Stable Diffusion model) generates class-conditioned samples via classifier-free guidance (scale 4.0, 50 steps). Mode Discovery: K-means (k=IPC) clusters real images' VAE-encoder latents per class into $N$ modes $\\mathbf{M_c}=\\{m_1,...,m_N\\}$ (any clustering algorithm works per the paper's own ablation). Mode Guidance: at each reverse step, a guidance vector $\\mathbf{g}_t=(m_i-\\hat{x}_0^t)$ (the direction from the model's current denoised-latent estimate toward the assigned target mode $m_i$) is added to the noise prediction, $\\hat{\\epsilon}_\\theta(x_t,t,c)=\\tilde\\epsilon_\\theta(x_t,t,c)+\\lambda\\cdot \\mathbf{g}_t\\cdot\\sigma_t$ (Eq. 6, $\\lambda{=}0.1$), steering that sample's trajectory toward its assigned mode. Stop Guidance: guidance is set to zero once the timestep falls below $t_{stop}{=}25$ (of 50 steps, i.e. roughly the boundary of the \"refinement stage\" per prior diffusion-stage analysis), letting the model finish denoising unconstrained to avoid the artifacts that full-trajectory guidance produces. No fine-tuning, no matching network, no expert trajectories, and (unlike D4M's TTM or SRe2L- family methods) no soft-label training-time matching is required by the core method, though soft labels are still used for the ImageNet-1K evaluation protocol.", "evidence": "ImageNette/ImageIDC, hard-label protocol, ResNet-10-avg-pool, IPC=10/20/50 (Table 1): DiT+MGD3 66.4%/ 71.2%/79.5% (Nette) vs. MinMaxDiff 62.0%/66.8%/76.6% vs. raw DiT 59.1%/64.8%/73.3% — +4.4/+4.4/+2.9 points over the prior SOTA the abstract cites; on IDC, MGD3 55.9%/61.9%/72.1% vs. MinMax 53.1%/59.0%/ 69.6%. ImageNet-100, hard-label, IPC=20 (Table 2): MGD3 beats MinMaxDiff by 1.3-1.6 points across ConvNet-6/ResNetAP-10/ResNet-18 (e.g. ResNetAP-10: 33.9% vs. 32.3%); at IPC=10, IDC-1 (a pixel-optimization method) edges out MGD3 slightly but at far higher cost. ImageNet-1K, soft-label (RDED-style region soft labels, ResNet-18 teacher/student), IPC=10/50: MGD3 beats prior SOTA by 1.3/1.6 points (Fig. 4d); scaling to larger student backbones at IPC=50 (Table 3): MGD3 ResNet-101 67.7% vs. EDC 64.9% vs. D4M 63.4% vs. RDED 61.2%. Generative-prior comparison, ImageNet-A-E, IPC=10, hard-label (Table 4): MGD3 beats GLaD/H-GLaD/LD3M (labeled \"LM3D\" in this paper) by a wide margin, e.g. ImageNet-A 63.4% vs. LD3M 57.0% vs. H-GLaD 55.1% vs. GLaD 53.1%. Off-target generator test: a general-purpose text-to-image Stable Diffusion model (not trained on any target dataset) with MGD3 guidance beats the same Stable Diffusion model without guidance by 3.4/2.3 points on ImageNet-1K IPC=10/50 (Fig. 4c), demonstrating the guidance mechanism transfers to a genuinely off-target generator, though absolute accuracy trails the ImageNet-trained-DiT variant. Cost: 0.42 GPU-hours to generate ImageNet-100 IPC=10 vs. Minimax's ~10 hours (fine-tuning) and IDC-1's >100 hours. Key ablation (Table 5, ImageNette IPC=10): Mode Discovery alone gives 53.2%/57.1%/53.5% (ConvNet-6/ResNetAP-10/ResNet-18); +Mode Guidance raises it to 57.5%/63.8%/62.0%; +Stop Guidance further to 59.6%/66.4%/64.4% — each of the three stages adds measurable accuracy, with Mode Guidance contributing the largest single jump. Sampler-agnosticism (Table 6): the method improves both DDPM and DDIM samplers, and both LDM (U-Net) and DiT (Transformer) backbones, similarly.", "limitations": "The headline numbers use a DiT pretrained on ImageNet-1K, i.e. matched to the target distribution for every ImageNet-derived experiment; the off-target Stable Diffusion test (Fig. 4c) is real but shows a smaller improvement and is reported only for ImageNet-1K, not for ImageNette/IDC/ImageNet-100, so its generality across datasets is untested; hard-label and soft-label protocols are used for different dataset groups within the same paper (ImageNette/IDC/ImageNet-100 hard, ImageNet-1K soft), so results are not directly comparable across the paper's own tables; mode discovery uses K-means with k fixed equal to IPC, which ties the number of distinct modes assumed to the budget rather than to the true number of underlying data clusters; the paper does not report results below IPC=10 or above IPC=100/ IPC=50 for ImageNet-1K's largest backbones." } }, { "id": "d3hr-2025", "type": "paper", "title": "Taming Diffusion for Dataset Distillation with High Representativeness", "short": "D3HR", "arxiv": "2505.18399", "url": null, "date": "2025-05-23", "venue": { "name": "ICML 2025", "status": "conference" }, "authors": [ "Lin Zhao", "Yushu Wu", "Xinru Jiang", "Jianyang Gu", "Yanzhi Wang", "Xiaolin Xu", "Pu Zhao", "Xue Lin" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "labels": "soft-relabel", "prior": [ "diffusion" ], "regime": "training-free", "scale": [ "imagenet-1k" ] }, "tier": "notable", "links": { "code": "https://github.com/lin-zhao-resoLve/D3HR", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Identifies inaccurate distribution matching, deviation from the random-noise domain, and separate per-image sampling as flaws in diffusion-based distillation, and fixes them with DDIM inversion that maps the full dataset's latents into a high-normality Gaussian domain, plus an efficient sampling scheme that aligns representative latents to that domain; reports higher cross-architecture accuracy than state-of-the-art diffusion-based baselines.", "abstract": "Recent deep learning models demand larger datasets, driving the need for dataset distillation to create compact, cost-efficient datasets while maintaining performance. Due to the powerful image generation capability of diffusion, it has been introduced to this field for generating distilled images. In this paper, we systematically investigate issues present in current diffusion-based dataset distillation methods, including inaccurate distribution matching, distribution deviation with random noise, and separate sampling. Building on this, we propose D^3HR, a novel diffusion-based framework to generate distilled datasets with high representativeness. Specifically, we adopt DDIM inversion to map the latents of the full dataset from a low-normality latent domain to a high-normality Gaussian domain, preserving information and ensuring structural consistency to generate representative latents for the distilled dataset. Furthermore, we propose an efficient sampling scheme to better align the representative latents with the high-normality Gaussian distribution. Our comprehensive experiments demonstrate that D^3HR can achieve higher accuracy across different model architectures compared with state-of-the-art baselines in dataset distillation. Source code: https://github.com/lin-zhao-resoLve/D3HR.", "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "explained": { "before": "D4M clusters real-image latents with K-means and Minimax fine-tunes a diffusion model with a cosine-similarity minimax objective to obtain representative latents; both still generate each distilled sample somewhat independently and rely on approximations (spherical K-means clusters, or direction-only cosine similarity) of the true latent distribution, which the paper's Lemma 3.1 argues is actually a complex multi-component Gaussian mixture, not something either approximation captures well. SRe2L/DWA/RDED need a teacher-architecture-specific synthesis step (BN statistics or patch selection), coupling the distilled set to one architecture.", "problem": "The paper diagnoses three specific flaws in prior diffusion-based distillation: (i) inaccurate distribution matching — the VAE latent space per class is a genuinely multi-component Gaussian mixture (proved in Appendix A.1), so K-means' spherical-cluster assumption or cosine-similarity's direction-only matching cannot describe it well; (ii) distribution deviation with random noise — generating from freshly sampled random noise (rather than from the real data's own noised trajectory) injects unpredictable randomness that can shift the denoised output away from the intended distribution; (iii) separate sampling — each of the n distilled latents per class is matched to the target distribution individually, so the *set* of n latents as a whole is not guaranteed to match the target distribution even if each element does reasonably well.", "idea": "Use DDIM inversion (not the stochastic DDPM forward process) to deterministically and bijectively map each real image's VAE latent into a noise-space domain that becomes provably closer to a single Gaussian as the number of inversion steps grows (Lemma 4.1); fit a per-class Gaussian to the inverted latents; then, instead of independently sampling n latents from that Gaussian, sample many candidate n-latent subsets and pick the one whose empirical mean/variance/skewness best match the fitted Gaussian (\"group sampling\"), before DDIM-sampling each selected latent back into an image.", "method": "A pretrained DiT + VAE (Peebles & Xie 2023, originally trained on ImageNet-1K) supplies the diffusion backbone; for CIFAR-10/100 and Tiny-ImageNet, this ImageNet-pretrained DiT is further fine-tuned for 400 epochs per dataset to adapt its generative capacity to the new label space and resolution (for ImageNet-1K itself, no fine-tuning is needed). For each class $\\mathcal{C}$: (1) Domain Mapping — every real image's VAE latent $z_0$ is deterministically inverted via DDIM ($T{=}31$ steps in the main experiments) to a noise-space latent $z_T$ (Eq. 3), forming $\\mathcal{Z}_{T,\\mathcal{C}}$; (2) Distribution Matching — the empirical mean $\\mu_{T,\\mathcal{C}}$ and variance $\\sigma^2_{T,\\mathcal{C}}$ of $\\mathcal{Z}_{T,\\mathcal{C}}$ define a fitted Gaussian $\\hat{\\mathcal{Z}}_{T,\\mathcal{C}}$; (3) Group Sampling — $m$ candidate subsets of $n{=}\\text{IPC}$ i.i.d. latents are drawn from $\\hat{\\mathcal{Z}}_{T,\\mathcal{C}}$ via the Ziggurat algorithm, each subset scored by a weighted sum $\\mathcal{L}_{T,\\mathcal{C}}=\\lambda_\\mu\\mathcal{L}_\\mu+\\lambda_\\sigma\\mathcal{L}_\\sigma+ \\lambda_{\\gamma_1}\\mathcal{L}_{\\gamma_1}$ of its mean/std/skewness deviation from the fitted Gaussian's parameters (Eqs. 5-9), and the lowest-scoring subset is kept; (4) each latent in the winning subset is DDIM-sampled back to $z_0$ and decoded to an image, paired with the class label. No architecture-specific matching or teacher model is used during synthesis at all — this is entirely training-free at the generation step. At evaluation, a separately pretrained teacher (ResNet-18, following SRe2L's convention) supplies soft labels to train the student, matching the label regime of SRe2L/DWA/D4M/RDED for the main comparison table; a separate section (6.4) additionally reports hard-label results matched to Minimax's own evaluation protocol on ImageWoof.", "evidence": "Table 1 (four datasets, ResNet-18/ResNet-101, soft labels from a ResNet-18 teacher, following SRe2L): ImageNet-1K IPC=10 D3HR 44.3% vs. RDED 42.0% vs. D4M 27.9% vs. SRe2L 21.3% (ResNet-18); IPC=50 D3HR 59.4% vs. RDED 56.5%; CIFAR-10 IPC=50 D3HR 70.8% vs. RDED 62.1% (+8.7, the largest gap in the table, cited as a \"12.5% relative improvement\" for ResNet-18 and \"17.4%\" for ResNet-101); Tiny-ImageNet IPC=10 D3HR 44.4% vs. RDED 41.9%. Cross-architecture on ImageNet-1K IPC=10 (Table 2, train-on-one/test-on- another grid across ResNet-18/MobileNet-V2/VGG-11): D3HR beats RDED in every one of 9 cells, e.g. trained on MobileNet-V2, tested on MobileNet-V2: D3HR 46.4% vs. RDED 33.8% vs. SRe2L 10.2%. Hard-label comparison with Minimax (Table 5, ImageWoof, matching Minimax's own protocol): D3HR beats Minimax at every IPC/architecture, e.g. IPC=50/ResNetAP-10: D3HR 59.3% vs. Minimax 56.3%. Cost: sampling (group sampling + DDIM decode) takes 2.6s/class on a single RTX A6000 for ImageNet-1K IPC=10 with $m{=}10^6$ candidate subsets; storage can be reduced to just the per-class Gaussian's mean/variance plus the shared DiT weights, smaller than storing all distilled latents/images. Key ablations (Table 3, ImageNet- 1K IPC=10, ResNet-18): domain mapping via DDIM inversion (Base-RS, 41.6%) beats DDPM-style random-noise domain mapping (Base-DDPM, 37.3%) by 4.3 points (paper's text calls this an \"11.5%\" relative gain), isolating the deterministic-inversion fix; adding group sampling with all three statistics ($\\mathcal{L}_\\mu+\\mathcal{L}_\\sigma+\\mathcal{L}_{\\gamma_1}$, 44.3%) beats Base-RS (individual Gaussian sampling, 41.6%) by 2.7 points and beats any single statistic alone (42.3-42.6%), showing the subset-level matching is a distinct, additive contribution on top of domain mapping; Figure 4 shows group sampling also reduces run-to-run variance versus individual random sampling.", "limitations": "The DiT backbone is either already ImageNet-1K-pretrained (used as-is for the ImageNet-1K experiments) or explicitly fine-tuned for 400 epochs per dataset (CIFAR-10/100, Tiny-ImageNet) — every reported number therefore uses a generator matched (by pretraining or fine-tuning) to the target distribution, with no off-target-generator ablation; the main results (Table 1) rely on teacher soft-label supervision at evaluation like its SRe2L-family baselines, so the paper does not fully separate generation-quality gains from label-regime effects except in the separate hard-label section (6.4), which is scoped only to ImageWoof/Minimax's setting; the single-Gaussian approximation (Lemma 4.1) is asymptotic in the number of inversion steps $T$, and the paper's own Fig. 5 shows accuracy peaks at $T{=}31$ and *declines* for larger $T$ because more inversion steps destroy structural information even as they improve the Gaussian fit — a stated trade-off, not a solved problem; excludes Minimax from the main Table 1 comparison (addressed separately) because Minimax \"requires training multiple diffusion models\" at large scale, a methodological choice the paper flags but that also avoids a fully matched head-to-head in the primary table." } }, { "id": "concord-2025", "type": "paper", "title": "CONCORD: Concept-Informed Diffusion for Dataset Distillation", "short": "CONCORD", "arxiv": "2505.18358", "url": null, "date": "2025-05-23", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Jianyang Gu", "Haonan Wang", "Ruoxi Jia", "Saeed Vahidian", "Vyacheslav Kungurtsev", "Wei Jiang", "Yiran Chen" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "prior": [ "diffusion", "llm" ] }, "tier": "catalogued", "links": { "code": "https://github.com/vimar-gu/CONCORD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Retrieves fine-grained, distinguishable concepts from a large language model for each class label and feeds them into the diffusion denoising process to correct missing or incorrect object detail at the instance level, addressing a gap left by prior generative distillation methods that match only dataset-level statistics. Reports state-of-the-art results on ImageNet-1K and its subsets without relying on a pretrained classifier.", "abstract": "Dataset distillation (DD) has witnessed significant progress in creating small datasets that encapsulate rich information from large original ones. Particularly, methods based on generative priors show promising performance, while maintaining computational efficiency and cross-architecture generalization. However, the generation process lacks explicit controllability for each sample. Previous distillation methods primarily match the real distribution from the perspective of the entire dataset, whereas overlooking concept completeness at the instance level. The missing or incorrectly represented object details cannot be efficiently compensated due to the constrained sample amount typical in DD settings. To this end, we propose incorporating the concept understanding of large language models (LLMs) to perform Concept-Informed Diffusion (CONCORD) for dataset distillation. Specifically, distinguishable and fine-grained concepts are retrieved based on category labels to inform the denoising process and refine essential object details. By integrating these concepts, the proposed method significantly enhances both the controllability and interpretability of the distilled image generation, without relying on pre-trained classifiers. We demonstrate the efficacy of CONCORD by achieving state-of-the-art performance on ImageNet-1K and its subsets. The code implementation is released in https://github.com/vimar-gu/CONCORD.", "notes": "" }, { "id": "dd-ranking-2025", "type": "paper", "title": "DD-Ranking: Rethinking the Evaluation of Dataset Distillation", "short": "DD-Ranking", "arxiv": "2505.13300", "url": null, "date": "2025-05-19", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Zekai Li", "Xinhao Zhong", "Samir Khaki", "Zhiyuan Liang", "Yuhao Zhou", "Mingjia Shi", "Ziqiao Wang", "Xuanlei Zhao", "Wangbo Zhao", "Ziheng Qin", "Mengxuan Wu", "Pengfei Zhou", "Haonan Wang", "David Junhao Zhang", "Jia-Wei Liu", "Shaobo Wang", "Dai Liu", "Linfeng Zhang", "Guang Li", "Kun Wang", "Zheng Zhu", "Zhiheng Ma", "Joey Tianyi Zhou", "Jiancheng Lv", "Yaochu Jin", "Peihao Wang", "Kaipeng Zhang", "Lingjuan Lyu", "Yiran Huang", "Zeynep Akata", "Zhiwei Deng", "Xindi Wu", "George Cazenavette", "Yuzhang Shang", "Justin Cui", "Jindong Gu", "Qian Zheng", "Hao Ye", "Shuo Wang", "Xiaobo Wang", "Yan Yan", "Angela Yao", "Mike Zheng Shou", "Tianlong Chen", "Hakan Bilen", "Baharan Mirzasoleiman", "Manolis Kellis", "Konstantinos N. Plataniotis", "Zhangyang Wang", "Bo Zhao", "Yang You", "Kai Wang" ], "kind": "evaluation-benchmark", "settings": [ "image-classification" ], "families": [], "facets": { "scale": [ "mnist-cifar", "tiny-imagenet", "imagenet-1k" ] }, "tier": "core", "links": { "code": "https://github.com/NUS-HPC-AI-Lab/DD-Ranking", "project": "https://nus-hpc-ai-lab.github.io/DD-Ranking/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "DC-BENCH (2022) standardized augmentation/architecture for hard-label matching methods (DC, DSA, DM, MTT), but by 2025 the field's frontier (SRe2L, RDED, CDA, DWA, D4M, EDC, G-VBSM, DATM) all train evaluation models with epoch-wise teacher soft labels and method-specific augmentation recipes (Table 1 catalogs wildly different epochs, batch sizes, optimizers and LR schedules across 13 methods), none of which DC-BENCH's protocol controls for.", "problem": "Test accuracy conflates three things that DD papers report as one number: the informativeness of the synthetic images, the knowledge-distillation effect of soft labels from a pretrained teacher, and the effect of the post-hoc data augmentation/training recipe. The paper's diagnosis, backed by controlled experiments, is that random real images annotated with the same soft labels often beat the actual distilled images (Section 2.2), and a substantial share of reported gains vanishes when augmentation is removed (Table 2, especially at ImageNet-1K resolution).", "idea": "Replace raw accuracy with two relative metrics that hold the confound fixed and vary only the data: a Label-Robust Score (LRS) that combines how much of the *hard-label* accuracy gap to the full dataset a method closes (Hard Label Recovery) with how much a method beats *random selection under its own label recipe* (Improvement over Random, IOR); and an Augmentation-Robust Score (ARS) that requires a method to beat random selection both with and without its augmentation.", "method": "No new synthesis method. Defines HLR = acc_real-hard - acc_syn-hard (smaller is better); IOR = acc_syn-any - acc_rdm-any, i.e. synthetic vs. a same-size random subset evaluated under the identical label type/recipe (larger is better, comparable across hard- and soft-label methods since both are relative to their own random-selection control); LRS = a normalized (via 100%*(e^alpha - e^-1)/(e - e^-1)) combination alpha = lambda*IOR - (1-lambda)*HLR, lambda=0.5 by default. ARS is defined analogously from IOR computed both with and without the method's augmentation (beta = gamma*IOR_aug + (1-gamma)*IOR_noaug). Benchmarks 13 methods (DC, DSA, MTT, DM, DataDAM as hard-label; SRe2L, DATM, EDF, DWA, RDED, CDA, EDC, G-VBSM as soft-label) on CIFAR-10, CIFAR-100, TinyImageNet and ImageNet-1K, each under its own reported architecture, then cross-checked on AlexNet/ResNet-50/VGG-11/Swin-Tiny/ViT-Base for robustness; 5 seeds per number.", "evidence": "ImageNet-1K, soft-label/relabel+KD regime (Table 6, Section \"Findings\" i): current DD methods \"consistently underperform random selection\" (negative IOR) across most IPC settings once random images are given the same soft labels, and the gap widens as IPC grows. CIFAR-10, IPC1 (Table 3, lambda=0.5): SRe2L has IOR = -1.9 (worse than random under its own soft-label recipe) and RDED IOR = -0.5, while hard-label MTT has IOR = +16.7 and LRS 23.1, the best on CIFAR-10; at IPC50, SRe2L IOR = -11.5, RDED IOR = -1.5 — soft-label decoupled methods lose to random selection at every tested IPC on CIFAR-10 despite far higher raw accuracy. Augmentation-robust score, ImageNet-1K (Table 7): most decoupled/generative methods (SRe2L, D4M, DWA, CDA, G-VBSM) show negative IOR both with and without augmentation at IPC50 (e.g. SRe2L: -13.2 with aug, -21.0 without), and the gap to random widens as augmentation is removed and as IPC grows, meaning their accuracy edge is substantially manufactured by the augmentation recipe, not the images. Cross-architecture stability (Fig. 2) and cross-teacher stability (Fig. 3): LRS varies little across 4 evaluation architectures and across different teacher models used for relabeling, validating LRS as a more stable metric than raw accuracy. Among soft-label methods, DATM/EDF (soft labels *jointly optimized* with images during trajectory matching) beat D4M/SRe2L/RDED (labels generated post hoc by a frozen teacher) on LRS — matching-based methods remain the strongest baselines overall despite not scaling to ImageNet-1K.", "limitations": "LRS and ARS still require access to the exact training recipe (optimizer, LR, epochs, augmentation) of each method to reproduce a matched random-selection control, so the benchmark is only as fair as each method's disclosed hyperparameters (Table 1 shows several methods use non-default optimizers without motivation). The paper reports ARS only for ImageNet-1K \"due to space limit,\" leaving CIFAR/TinyImageNet augmentation-robustness less thoroughly documented. LRS's normalization (lambda=0.5 default) is a design choice; the appendix explores other lambda values (Tables 10-19) but the main paper does not argue lambda=0.5 is uniquely correct. Matching-based (hard-label) methods score best on LRS but the paper does not resolve whether this reflects genuinely more informative images or simply that hard-label evaluation has a smaller ceiling to close (full-dataset hard-label accuracy is lower, so HLR is easier to shrink). Does not evaluate coreset-selection methods (K-Center, forgetting) as first-class baselines with LRS/ARS, only as the \"random\" control." }, "summary": "Shows that reported accuracy gains from decoupled dataset-distillation pipelines largely come from soft labels and stronger post-hoc augmentation rather than the intrinsic quality of the distilled images themselves, since even randomly sampled images can match state-of-the-art distilled sets under the same labeling and augmentation recipe, and proposes DD-Ranking, a unified evaluation framework and set of metrics that isolate the images' own contribution.", "abstract": "In recent years, dataset distillation has provided a reliable solution for data compression, where models trained on the resulting smaller synthetic datasets achieve performance comparable to those trained on the original datasets. To further improve the performance of synthetic datasets, various training pipelines and optimization objectives have been proposed, greatly advancing the field of dataset distillation. Recent decoupled dataset distillation methods introduce soft labels and stronger data augmentation during the post-evaluation phase and scale dataset distillation up to larger datasets (e.g., ImageNet-1K). However, this raises a question: Is accuracy still a reliable metric to fairly evaluate dataset distillation methods? Our empirical findings suggest that the performance improvements of these methods often stem from additional techniques rather than the inherent quality of the images themselves, with even randomly sampled images achieving superior results. Such misaligned evaluation settings severely hinder the development of DD. Therefore, we propose DD-Ranking, a unified evaluation framework, along with new general evaluation metrics to uncover the true performance improvements achieved by different methods. By refocusing on the actual information enhancement of distilled datasets, DD-Ranking provides a more comprehensive and fair evaluation standard for future research advancements.", "awesome_sections": [ "Main / Benchmark" ], "notes": "" }, { "id": "repblend-2025", "type": "paper", "title": "Beyond Modality Collapse: Representations Blending for Multimodal Dataset Distillation", "short": "RepBlend", "arxiv": "2505.14705", "url": null, "date": "2025-05-16", "venue": { "name": "NeurIPS 2025", "status": "conference" }, "authors": [ "Xin Zhang", "Ziruo Zhang", "Jiawei Du", "Zuozhu Liu", "Joey Tianyi Zhou" ], "kind": "method", "settings": [ "vision-language" ], "families": [ "trajectory-matching" ], "facets": { "labels": "similarity-matrix", "prior": [ "trained-experts" ], "regime": "bilevel", "parameterization": [ "pixels", "embedding-features", "similarity-matrix" ], "scale": [ "flickr30k-coco" ] }, "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-14", "source": "arxiv-api", "depth": "full-text" }, "summary": "RepBlend identifies \"modality collapse\" in multimodal dataset distillation, over-concentrated intra-modal representations and an enlarged cross-modal gap, as a conflict between dataset distillation's over-compression and contrastive cross-modal supervision, and addresses it by weakening overdominant cross-modal supervision through representation blending to restore intra-modal diversity, plus symmetric projection trajectory matching that synchronizes optimization dynamics across modality-specific projection heads to correct asymmetric supervision. On Flickr30K and MS-COCO the paper reports consistent gains over prior multimodal distillation methods (e.g. +9.4 IR@10, +6.3 TR@10 at the 100-pair setting) with up to 6.7x faster distillation.", "abstract": "Multimodal Dataset Distillation (MDD) seeks to condense large-scale image-text datasets into compact surrogates while retaining their effectiveness for cross-modal learning. Despite recent progress, existing MDD approaches often suffer from \\textit{\\textbf{Modality Collapse}}, characterized by over-concentrated intra-modal representations and enlarged distributional gap across modalities. In this paper, at the first time, we identify this issue as stemming from a fundamental conflict between the over-compression behavior inherent in dataset distillation and the cross-modal supervision imposed by contrastive objectives. To alleviate modality collapse, we introduce \\textbf{RepBlend}, a novel MDD framework that weakens overdominant cross-modal supervision via representation blending, thereby significantly enhancing intra-modal diversity. Additionally, we observe that current MDD methods impose asymmetric supervision across modalities, resulting in biased optimization. To address this, we propose symmetric projection trajectory matching, which synchronizes the optimization dynamics using modality-specific projection heads, thereby promoting balanced supervision and enhancing cross-modal alignment. Experiments on Flickr-30K and MS-COCO show that RepBlend consistently outperforms prior state-of-the-art MDD methods, achieving significant gains in retrieval performance (e.g., +9.4 IR@10, +6.3 TR@10 under the 100-pair setting) and offering up to 6.7$\\times$ distillation speedup.", "builds_on": [ "lors-2024", "mtt-vl-2024" ], "explained": { "before": "LoRS and MTT-VL established trajectory matching for image-text pairs, with LoRS adding a distilled similarity matrix (via weighted BCE) so synthetic pairs carry soft ground-truth affinities instead of an assumed one-to-one match. Both keep separate, asymmetric treatment of the two modalities: the image tower's full encoder is matched while the text side is largely a frozen backbone plus a light projection.", "problem": "The paper diagnoses \"modality collapse\": as distillation compresses data, intra-modal representations become increasingly concentrated (cosine similarity among synthetic embeddings of the same modality rises over training) while the cross-modal gap between image and text embeddings widens. It traces this to a structural conflict between dataset distillation's over-compression (which pulls a few synthetic points to represent many real ones) and contrastive cross-modal supervision (whose positive-pair gradients pull embeddings toward each other's modality, over-concentrating each modality's own spread) - formalized with a gradient analysis (their Eq. 3) and measured empirically (concentration ratio rising over 3000 iterations).", "idea": "Counteract the contrastive objective's concentrating pressure by blending synthetic representations across instances (representation blending), which perturbs optimization away from collapse while preserving semantic structure; and replace prior methods' asymmetric trajectory matching (full image encoder vs. text projection only) with symmetric projection trajectory matching, matching lightweight projection heads on both modalities so neither is over- or under-supervised, which also removes the need to match a full encoder trajectory.", "method": "Synthetic image pixels, synthetic 768-d text embeddings, and soft pairwise labels are directly optimized parameters. Both an image encoder (NFNet/ResNet-50/ViT) and a text encoder (BERT/DistilBERT) are pretrained and kept frozen during distillation; only a linear projection head per modality is trained/matched. The base objective is the weighted binary cross-entropy (wBCE) of LoRS between predicted cosine similarities and soft labels. Representation blending (Eq. 5) linearly interpolates a synthetic embedding with a randomly shuffled instance's embedding ($\\lambda\\sim\\text{Beta}(\\alpha,\\alpha)$) before the projection head, done independently per modality, to reduce intra-modal concentration. Symmetric projection trajectory matching (Eq. 6) matches the image-projection and text-projection parameter trajectories jointly and symmetrically after $T$ synthetic steps against $M$ real steps, normalized by the expert's total movement in both projections combined, replacing the asymmetric image-encoder/text-projection matching of MTT-VL and LoRS.", "evidence": "With NFNet image encoder and BERT text encoder, Flickr30K at 100 pairs: IR@1/5/10 = 11.5/32.0/44.5, TR@1/5/10 = 16.2/41.7/55.5 (Table 1), versus LoRS's re-run 8.3/24.1/35.1 and 11.8/35.8/49.2 at the same budget; at 500 pairs IR@1/5/10 = 17.0/42.5/55.9, TR@1/5/10 = 22.5/53.2/66.7 versus LoRS 10.0/28.9/41.6 and 15.5/39.8/53.7. On COCO at 100/500 pairs (Table 2): IR@1 = 4.1/6.2, TR@1 = 5.2/7.0, versus LoRS 1.8/2.8 and 3.3/5.3. Baselines (coreset methods, MTT-VL, TESLA-VL, LoRS) are re-run under the same 20-expert-trajectory buffer and hyperparameters (Tables 5-6). Cost (Table 4, single comparison against LoRS): buffer generation 40 vs 70 min/trajectory (1.75x), buffer memory 0.73 vs 1.63 GB (2.23x), distillation step time 1.71 vs 11.5 s/iteration (6.7x), peak VRAM 10.17 vs 21.78 GB (2.14x). The key ablation (Figure 5) removes representation blending or symmetric projection matching individually; both individually degrade IR/TR at every budget on Flickr30K relative to the full method, with the combination needed for the best numbers. A cross-architecture ablation (Table 3, distilled on NFNet+BERT, evaluated on ResNet-50+BERT and RegNet+BERT at 500 pairs) shows RepBlend transfers better than LoRS, e.g. ResNet-50 IR@10 23.6% vs 20.4%.", "limitations": "The paper states current MDD frameworks (including this one) remain pair-level: they do not model fine-grained token-to-object correspondence within a pair or cross-instance interactions beyond the blending step. All experiments use frozen, ImageNet/BERT-pretrained encoders, so results are a property of that backbone pair as much as of the method. Evaluation is centered on Flickr30K and MS-COCO; an audio-text (AudioCaps) extension is included but only briefly. An appendix comparison to a unimodal decoupled method (SRe2L) on ImageNet-100 shows a very large gap that the authors note is not a fair comparison since the setups differ. The claimed 6.7x speedup is for the distillation step only, not buffer/expert-trajectory generation, which is cut by a smaller 1.75x." }, "notes": "" }, { "id": "unidetox-2025", "type": "paper", "title": "UniDetox: Universal Detoxification of Large Language Models via Dataset Distillation", "short": "UniDetox", "arxiv": "2504.20500", "url": null, "date": "2025-04-29", "venue": { "name": "ICLR 2025", "status": "conference" }, "authors": [ "Huimin Lu", "Masaru Isonuma", "Junichiro Mori", "Ichiro Sakata" ], "kind": "trustworthy", "settings": [ "text" ], "families": [], "tier": "notable", "links": { "code": "https://github.com/EminLU/UniDetox", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "UniDetox distills a small synthetic text dataset that carries detoxifying signal, generated from one source model (GPT-2) via contrastive decoding, and fine-tunes any target LLM on it to reduce toxicity without per-model hyperparameter tuning. The paper reports that text distilled from GPT-2 successfully detoxifies larger, unrelated models (OPT, Falcon, LLaMA-2) with a single shared hyperparameter configuration, and that the distilled text also reduces politically biased content.", "abstract": "We present UniDetox, a universally applicable method designed to mitigate toxicity across various large language models (LLMs). Previous detoxification methods are typically model-specific, addressing only individual models or model families, and require careful hyperparameter tuning due to the trade-off between detoxification efficacy and language modeling performance. In contrast, UniDetox provides a detoxification technique that can be universally applied to a wide range of LLMs without the need for separate model-specific tuning. Specifically, we propose a novel and efficient dataset distillation technique for detoxification using contrastive decoding. This approach distills detoxifying representations in the form of synthetic text data, enabling universal detoxification of any LLM through fine-tuning with the distilled text. Our experiments demonstrate that the detoxifying text distilled from GPT-2 can effectively detoxify larger models, including OPT, Falcon, and LLaMA-2. Furthermore, UniDetox eliminates the need for separate hyperparameter tuning for each model, as a single hyperparameter configuration can be seamlessly applied across different models. Additionally, analysis of the detoxifying text reveals a reduction in politically biased content, providing insights into the attributes necessary for effective detoxification of LLMs.", "notes": "Families left empty: the synthetic text is produced via contrastive decoding, not a P1-P4 matching objective stated in the abstract." }, { "id": "latent-video-dataset-distillation-2025", "type": "paper", "title": "Latent Video Dataset Distillation", "arxiv": "2504.17132", "url": null, "date": "2025-04-23", "venue": { "name": "CVPR 2025 Workshop", "status": "workshop" }, "authors": [ "Ning Li", "Antai Andy Liu", "Jingran Zhang", "Justin Cui" ], "kind": "method", "settings": [ "video" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "generative-latent" ] }, "tier": "catalogued", "links": { "code": "https://github.com/liningresearch/Latent_Video_Dataset_Distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "This method stores condensed video data as codes in the latent space of a state-of-the-art variational encoder, rather than in pixel space, and picks which latents to keep with a diversity-aware selection strategy plus a training-free compression step on top. The paper reports new state-of-the-art results on all tested video-distillation benchmarks, including a 2.6-point gain on HMDB51 at IPC 1 and a 7.8-point gain on MiniUCF at IPC 5.", "abstract": "Dataset distillation has demonstrated remarkable effectiveness in high-compression scenarios for image datasets. While video datasets inherently contain greater redundancy, existing video dataset distillation methods primarily focus on compression in the pixel space, overlooking advances in the latent space that have been widely adopted in modern text-to-image and text-to-video models. In this work, we bridge this gap by introducing a novel video dataset distillation approach that operates in the latent space using a state-of-the-art variational encoder. Furthermore, we employ a diversity-aware data selection strategy to select both representative and diverse samples. Additionally, we introduce a simple, training-free method to further compress the distilled latent dataset. By combining these techniques, our approach achieves a new state-of-the-art performance in dataset distillation, outperforming prior methods on all datasets, e.g. on HMDB51 IPC 1, we achieve a 2.6% performance increase; on MiniUCF IPC 5, we achieve a 7.8% performance increase. Our code is available at https://github.com/liningresearch/Latent_Video_Dataset_Distillation.", "notes": "Selection-plus-latent-storage method rather than a matching optimization; classified under parameterization for the storage change, with the diversity-aware selection left unassigned to a P0-P4 family." }, { "id": "knowledge-distillation-and-dataset-2025", "type": "paper", "title": "Knowledge Distillation and Dataset Distillation of Large Language Models: Emerging Trends, Challenges, and Future Directions", "arxiv": "2504.14772", "url": null, "date": "2025-04-20", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Luyang Fang", "Xiaowei Yu", "Jiazhang Cai", "Yongkai Chen", "Shushan Wu", "Zhengliang Liu", "Zhenyuan Yang", "Haoran Lu", "Xilin Gong", "Yufang Liu", "Terry Ma", "Wei Ruan", "Ali Abbasi", "Jing Zhang", "Tao Wang", "Ehsan Latif", "Weihang You", "Hanqi Jiang", "Wei Liu", "Wei Zhang", "Soheil Kolouri", "Xiaoming Zhai", "Dajiang Zhu", "Wenxuan Zhong", "Tianming Liu", "Ping Ma" ], "kind": "survey", "settings": [ "text" ], "families": [], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Surveys knowledge distillation (KD) and dataset distillation (DD) as complementary strategies for compressing large language models. Covers KD methodologies (task-specific alignment, rationale-based training, multi-teacher frameworks) alongside DD techniques that synthesize compact fine-tuning data via optimization-based gradient matching, latent-space regularization, and generative synthesis, discusses combining the two, surveys applications in healthcare and education, and identifies open challenges in preserving emergent reasoning and linguistic diversity and in establishing evaluation protocols.", "abstract": "The exponential growth of Large Language Models (LLMs) continues to highlight the need for efficient strategies to meet ever-expanding computational and data demands. This survey provides a comprehensive analysis of two complementary paradigms: Knowledge Distillation (KD) and Dataset Distillation (DD), both aimed at compressing LLMs while preserving their advanced reasoning capabilities and linguistic diversity. We first examine key methodologies in KD, such as task-specific alignment, rationale-based training, and multi-teacher frameworks, alongside DD techniques that synthesize compact, high-impact datasets through optimization-based gradient matching, latent space regularization, and generative synthesis. Building on these foundations, we explore how integrating KD and DD can produce more effective and scalable compression strategies. Together, these approaches address persistent challenges in model scalability, architectural heterogeneity, and the preservation of emergent LLM abilities. We further highlight applications across domains such as healthcare and education, where distillation enables efficient deployment without sacrificing performance. Despite substantial progress, open challenges remain in preserving emergent reasoning and linguistic diversity, enabling efficient adaptation to continually evolving teacher models and datasets, and establishing comprehensive evaluation protocols. By synthesizing methodological innovations, theoretical foundations, and practical insights, our survey charts a path toward sustainable, resource-efficient LLMs through the tighter integration of KD and DD principles.", "notes": "Kept: dataset distillation is a co-equal, substantial half of the survey's scope, not a passing mention." }, { "id": "gps-2025", "type": "paper", "title": "GPS: Distilling Compact Memories via Grid-based Patch Sampling for Efficient Online Class-Incremental Learning", "short": "GPS", "arxiv": "2504.10409", "url": null, "date": "2025-04-14", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Mingchuan Ma", "Yuhao Zhou", "Jindi Lv", "Yuxin Tian", "Dan Si", "Shujian Li", "Qing Ye", "Jiancheng Lv" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "downsampled-multiformation" ], "regime": "training-free" }, "applications": [ "continual-learning" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "For online class-incremental replay, generates memory samples by sampling a grid of pixels from each real image into a compact low-resolution representation, with no trainable model or optimization, then reassembles the grid at replay time — avoiding the bi-level optimization cost of prior distilled-memory replay methods. Reports 3-4 percentage-point average end-accuracy gains when substituted into existing replay frameworks under memory-constrained settings, with limited computational overhead.", "abstract": "Online class-incremental learning aims to enable models to continuously adapt to new classes with limited access to past data, while mitigating catastrophic forgetting. Replay-based methods address this by maintaining a small memory buffer of previous samples, achieving competitive performance. For effective replay under constrained storage, recent approaches leverage distilled data to enhance the informativeness of memory. However, such approaches often involve significant computational overhead due to the use of bi-level optimization. Motivated by these limitations, we introduce Grid-based Patch Sampling (GPS), a lightweight and effective strategy for distilling informative memory samples without relying on a trainable model. GPS generates informative samples by sampling a subset of pixels from the original image, yielding compact low-resolution representations that preserve both semantic content and structural information. During replay, these representations are reassembled to support training and evaluation. Experiments on extensive benchmarks demonstrate that GRS can be seamlessly integrated into existing replay frameworks, leading to 3%-4% improvements in average end accuracy under memory-constrained settings, with limited computational overhead.", "notes": "" }, { "id": "dataset-distillation-of-3d-2025", "type": "paper", "title": "Dataset Distillation of 3D Point Clouds via Distribution Matching", "short": "SADM", "arxiv": "2503.22154", "url": null, "date": "2025-03-28", "venue": { "name": "NeurIPS 2025", "status": "conference" }, "authors": [ "Jae-Young Yim", "Dongwook Kim", "Jae-Young Sim" ], "kind": "method", "settings": [ "other-data" ], "families": [ "distribution-matching" ], "tier": "notable", "links": { "code": "https://github.com/donguk071/SADM", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Extends distribution matching to 3D point-cloud dataset distillation, jointly optimizing synthetic point geometry and orientation; introduces a Semantically Aligned Distribution Matching loss computed on per-channel sorted features to correct the semantic misalignment caused by unordered point indexing, and jointly learns rotation angles to correct orientation variation, reporting consistent gains over prior distillation methods with strong cross-architecture generalization.", "abstract": "Large-scale datasets are usually required to train deep neural networks, but it increases the computational complexity hindering the practical applications. Recently, dataset distillation for images and texts has been attracting a lot of attention, that reduces the original dataset to a synthetic dataset to alleviate the computational burden of training while preserving essential task-relevant information. However, the dataset distillation for 3D point clouds remains largely unexplored, as the point clouds exhibit fundamentally different characteristics from that of images, making the dataset distillation more challenging. In this paper, we propose a distribution matching-based distillation framework for 3D point clouds that jointly optimizes the geometric structures as well as the orientations of the synthetic 3D objects. To address the semantic misalignment caused by unordered indexing of points, we introduce a Semantically Aligned Distribution Matching loss computed on the sorted features in each channel. Moreover, to address the rotation variation, we jointly learn the optimal rotation angles while updating the synthetic dataset to better align with the original feature distribution. Extensive experiments on widely used benchmark datasets demonstrate that the proposed method consistently outperforms existing dataset distillation methods, achieving superior accuracy and strong cross-architecture generalization.", "awesome_sections": [ "Applications / Point Cloud" ], "notes": "" }, { "id": "nrr-dd-2025", "type": "paper", "title": "Enhancing Dataset Distillation via Non-Critical Region Refinement", "short": "NRR-DD", "arxiv": "2503.18267", "url": null, "date": "2025-03-24", "venue": { "name": "CVPR 2025", "status": "conference" }, "authors": [ "Minh-Tuan Tran", "Trung Le", "Xuan-May Le", "Thanh-Toan Do", "Dinh Phung" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "facets": { "labels": "mixed", "prior": [ "pretrained-classifier" ], "regime": "selection", "parameterization": [ "pixels" ], "scale": [ "imagenet-1k" ] }, "tier": "catalogued", "links": { "code": "https://github.com/tmtuan1307/NRR-DD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "SRe2L updates every pixel of a synthetic image by inverting BN statistics, capturing class-general information but losing fine-grained detail. RDED instead selects and stitches high-confidence real crops (via a pretrained observer), capturing instance-specific detail but with no further refinement toward class-general structure. Both store dense per-crop soft labels (SRe2L: up to 120 GB for ImageNet-1K at IPC=200), a cost the paper frames as a separate open problem.", "problem": "Methods that refine all pixels toward class-general features (SRe2L-style) discard instance- specific detail; methods that select real patches without refinement (RDED) capture instance detail but no class-general structure. The paper diagnoses this as an unforced binary choice, and separately diagnoses the soft-label storage cost of every prior relabel+KD method as needlessly large, since a full 1,000-dimensional soft-label vector is stored per crop when only a scalar signal is later used to compute the KD loss.", "idea": "Use Class Activation Mapping to separate an image into critical (instance-specific) and non- critical (safe-to-change) regions, and update only the non-critical pixels toward class-general statistics, leaving the critical, patch-selected content untouched. Separately, replace the stored soft-label vector with just two scalar cross-entropy distances (from the mixed image's soft label to each of the two source images' one-hot labels), reasoning that only those two distances are needed to reconstruct the KD-style training signal.", "method": "Three stages, one pretrained teacher $\\mathcal{T}$, no student in the loop. (1) Critical-based Initial Data Discovery: compute per-image CAM, crop into patches, keep the top-$t$ highest-CAM-value patches in a pool; unlike RDED (which keeps the highest-confidence patches), NRR-DD selects the $g=\\beta\\times\\text{IPC}$ *lowest*-confidence patches from that pool (the \"hardest\" examples) and combines $\\beta$ of them per synthetic image, as in RDED's stitching. (2) Non-Critical Region Refinement: build a per-pixel mask $M=\\max\\{0,\\epsilon-C\\}$ from the CAM matrix $C$ (near-zero on high-CAM pixels, larger on low-CAM pixels) and update the image only where $M>0$: $\\tilde x \\leftarrow \\tilde x - M\\times\\eta\\nabla_{\\tilde x}\\mathcal{L}_C$, with $\\mathcal{L}_C=\\mathcal{L}_{ce}(\\mathcal{T}(\\tilde x),\\tilde y)+\\alpha_{bn}\\mathcal{L}_{bn}( \\mathcal{T}(\\tilde x))$ (cross-entropy plus SRe2L-style BN-statistics matching). (3) Distance-Based Representative (DBR) knowledge transfer: for a CutMix/Mixup pair $(\\tilde x_{org}, \\tilde x_{aug})$, compute the teacher's soft label on the mixed image and store only $d^T_{org}=\\mathcal{L}_{ce}( \\tilde y_{soft},\\tilde y_{org})$ and $d^T_{aug}=\\mathcal{L}_{ce}(\\tilde y_{soft},\\tilde y_{aug})$ (two scalars, not a full soft-label vector); the student is trained by matching its own two analogous distances to these stored targets (Eq. 9), optionally with an added label-refinement term $\\mathcal{L}_{lr}$.", "evidence": "ImageNet-1K, full soft-label relabel+KD regime (Table 1, IPC values as columns): ResNet-18, IPC=10 NRR-DD 46.1% vs RDED 42.0%, SRe2L 21.3%; IPC=50 NRR-DD 60.2% vs RDED 56.5%, SRe2L 46.8%. ResNet-101, IPC=10 NRR-DD 51.3% vs RDED 48.3%, SRe2L 30.9%; IPC=50 NRR-DD 64.3% vs RDED 61.2%, SRe2L 60.8%. Genuine hard-label and compressed-label results (Table 2, ImageNet-1K, ResNet-18, IPC=10): RDED with soft labels (SL) 42.0%, RDED with one-hot hard labels (OH) 16.3%, RDED with 2-element compact labels (CL) 22.2%, NRR-DD with DBR alone (no label refinement) 34.3%, NRR-DD with DBR+label- refinement 37.2% -- a \"Recover rate\" (DBR-OneHot)/(SoftLabel-OneHot) of 70%, i.e. DBR recovers 70% of the soft-label-vs-hard-label accuracy gap while storing only 2 scalars per crop instead of 1,000 (IPC=200 ImageNet-1K: 0.2 GB vs 120 GB for full soft labels, a ~500x reduction, stated in the abstract/intro and demonstrated via the Table 2 recover-rate results at IPC=10/50). At IPC=1, DBR (no soft label at all) already beats RDED-with-soft-labels on several datasets (stated in Section 4.3). Cost (Table 11, ResNet-18, 100 images on ImageNet-1K): NRR-DD 520.65 ms vs SRe2L 2113.23 ms and RDED 39.89 ms; peak memory 9.14 GB, matching SRe2L, versus RDED's 1.57 GB -- refinement costs more than RDED's selection-only pipeline but far less than full BN-statistics inversion from noise. Key ablation (Table 4, referenced in text): CIDD alone (patch selection only, no refinement) already beats RDED; adding NRR on top adds roughly a further 15-point gain on CIFAR-10/ResNet-18, isolating the refinement step from the selection step.", "limitations": "Stated: performance depends on CAM quality, which is sensitive to how the teacher itself was trained, and the paper does not explore improving CAMs directly; DBR's memory savings trade off against fine-grained-detail preservation on some complex tasks, left unresolved. Observed: NRR-DD with full soft labels still requires the same per-crop teacher relabeling infrastructure as SRe2L/ RDED to build its Table 1 numbers -- DBR is presented as an alternative, not a replacement, and the paper's headline SOTA claims (Table 1) use full soft labels, not DBR; DBR's own accuracy (34.3-37.2% at ImageNet-1K IPC=10) remains meaningfully below the soft-label number (46.1%), so it recovers most, not all, of the label information; synthesis is slower and more memory-hungry than RDED, the method it is closest to and builds on." }, "summary": "Enriches synthetic images' non-critical regions with class-general information while preserving instance-specific fine-grained detail elsewhere, instead of the common all-or-nothing choice between the two feature types; also introduces Distance-Based Representative knowledge transfer, which trains without soft labels by using the distance between synthetic-data predictions and one-hot labels, storing only two distances per instance while remaining competitive across small- and large-scale settings.", "abstract": "Dataset distillation has become a popular method for compressing large datasets into smaller, more efficient representations while preserving critical information for model training. Data features are broadly categorized into two types: instance-specific features, which capture unique, fine-grained details of individual examples, and class-general features, which represent shared, broad patterns across a class. However, previous approaches often struggle to balance these features-some focus solely on class-general patterns, neglecting finer instance details, while others prioritize instance-specific features, overlooking the shared characteristics essential for class-level understanding. In this paper, we introduce the Non-Critical Region Refinement Dataset Distillation (NRR-DD) method, which preserves instance-specific details and fine-grained regions in synthetic data while enriching non-critical regions with class-general information. This approach enables models to leverage all pixel information, capturing both feature types and enhancing overall performance. Additionally, we present Distance-Based Representative (DBR) knowledge transfer, which eliminates the need for soft labels in training by relying on the distance between synthetic data predictions and one-hot encoded labels. Experimental results show that NRR-DD achieves state-of-the-art performance on both small- and large-scale datasets. Furthermore, by storing only two distances per instance, our method delivers comparable results across various settings. The code is available at https://github.com/tmtuan1307/NRR-DD.", "builds_on": [ "sre2l-2023", "rded-2024" ], "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "ccfs-2025", "type": "paper", "title": "Curriculum Coarse-to-Fine Selection for High-IPC Dataset Distillation", "short": "CCFS", "arxiv": "2503.18872", "url": null, "date": "2025-03-24", "venue": { "name": "CVPR 2025", "status": "conference" }, "authors": [ "Yanda Chen", "Gongwei Chen", "Miao Zhang", "Weili Guan", "Liqiang Nie" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/CYDaaa30/CCFS", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Addresses the accuracy decay of dataset distillation at high images-per-class by curriculum-selecting real images to combine with the synthetic set at each stage, coarse-to-fine and conditioned on the current synthetic data rather than choosing real data once and independently, reporting gains of up to 6.6 points on CIFAR-10 and 5.8 on CIFAR-100 over prior real+synthetic combination methods at high IPC.", "abstract": "Dataset distillation (DD) excels in synthesizing a small number of images per class (IPC) but struggles to maintain its effectiveness in high-IPC settings. Recent works on dataset distillation demonstrate that combining distilled and real data can mitigate the effectiveness decay. However, our analysis of the combination paradigm reveals that the current one-shot and independent selection mechanism induces an incompatibility issue between distilled and real images. To address this issue, we introduce a novel curriculum coarse-to-fine selection (CCFS) method for efficient high-IPC dataset distillation. CCFS employs a curriculum selection framework for real data selection, where we leverage a coarse-to-fine strategy to select appropriate real data based on the current synthetic dataset in each curriculum. Extensive experiments validate CCFS, surpassing the state-of-the-art by +6.6\\% on CIFAR-10, +5.8\\% on CIFAR-100, and +3.4\\% on Tiny-ImageNet under high-IPC settings. Notably, CCFS achieves 60.2\\% test accuracy on ResNet-18 with a 20\\% compression ratio of Tiny-ImageNet, closely matching full-dataset training with only 0.3\\% degradation. Code: https://github.com/CYDaaa30/CCFS.", "awesome_sections": [ "Main / Better Optimization" ] }, { "id": "condensing-action-segmentation-datasets-2025", "type": "paper", "title": "Condensing Action Segmentation Datasets via Generative Network Inversion", "arxiv": "2503.14112", "url": null, "date": "2025-03-18", "venue": { "name": "CVPR 2025", "status": "conference" }, "authors": [ "Guodong Ding", "Rongyu Chen", "Angela Yao" ], "kind": "method", "settings": [ "video" ], "families": [ "generative-latent-optimization" ], "facets": { "parameterization": [ "generative-latent" ] }, "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Condenses procedural video datasets for temporal action segmentation into compact latent codes using a generative prior learned from the dataset and network inversion, reducing storage across both the temporal and channel dimensions, plus a diversity-driven sampling step that selects representative action sequences to cut video-wise redundancy; on the Breakfast dataset it reports over 500x storage reduction while retaining 83% of full-dataset segmentation performance, and improves downstream incremental-learning performance.", "abstract": "This work presents the first condensation approach for procedural video datasets used in temporal action segmentation. We propose a condensation framework that leverages generative prior learned from the dataset and network inversion to condense data into compact latent codes with significant storage reduced across temporal and channel aspects. Orthogonally, we propose sampling diverse and representative action sequences to minimize video-wise redundancy. Our evaluation on standard benchmarks demonstrates consistent effectiveness in condensing TAS datasets and achieving competitive performances. Specifically, on the Breakfast dataset, our approach reduces storage by over 500$\\times$ while retaining 83% of the performance compared to training with the full dataset. Furthermore, when applied to a downstream incremental learning task, it yields superior performance compared to the state-of-the-art.", "awesome_sections": [ "Applications / Video" ], "notes": "" }, { "id": "ddif-2025", "type": "paper", "title": "Distilling Dataset into Neural Field", "short": "DDiF", "arxiv": "2503.04835", "url": null, "date": "2025-03-05", "venue": { "name": "ICLR 2025", "status": "conference" }, "authors": [ "Donghyeok Shin", "HeeSun Bae", "Gyuwon Sim", "Wanmo Kang", "Il-Chul Moon" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "neural-field" ], "labels": "hard", "prior": [ "none" ], "regime": "single-level", "scale": [ "imagenet-subsets", "large-scale-other" ] }, "tier": "core", "links": { "code": "https://github.com/aailab-kaist/DDiF", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "builds_on": [ "idc-2022", "fred-2023", "haba-2022", "rtp-2022", "mtt-2022" ], "explained": { "before": "Static decoders (idc-2022's bilinear upsampling, fred-2023's inverse frequency transform) are fast and training-free but fixed, which the paper argues caps expressiveness; parameterized decoders (haba-2022, rtp-2022) are more flexible but the decoder's own parameters consume storage budget, and deep generative priors (e.g. glad-family latent optimization) need a large pretrained generator and slow backward passes.", "problem": "All prior decoding functions map a low-dimensional code into a fixed, instance-sized output, so their representational capacity is tied to the data dimension D: as D grows (high resolution, video, 3D), the decoder or code must grow too, which is exactly the scalability problem parameterization is meant to solve, and no prior parameterization method has a theoretical account of its own expressiveness.", "idea": "Store each synthetic instance as a small neural network (a neural field) that maps a coordinate to a value, rather than as a code plus decoder tied to the instance's shape; because a neural field's parameter count is independent of the output resolution and grid dimensionality, the same budget-constrained network can decode images, video frames, audio samples or 3D voxels by simply changing the input coordinate set, and (proven via a feasible-space argument) achieves a strictly larger feasible output space than fred-2023 under an equal per-instance parameter budget.", "method": "A coordinate set $\\mathcal{C}$ (the lattice of pixel/voxel/time-sample locations) is fixed and never optimized or stored. Each synthetic instance is one small L-layer sine-activated MLP $F_\\psi$ (a SIREN); a decoded instance is $\\tilde x=F_\\psi(\\mathcal{C})$. The synthetic set is $\\mathcal{S}=\\{(F_{\\psi_j} (\\mathcal{C}),\\tilde y_j)\\}_{j=1}^{|\\Psi|}$, trained end-to-end under any distillation loss (default: trajectory matching), with a warm-up step that fits each field to a randomly sampled real instance before distillation begins. Budget is the number of parameters $b$ per field (independent of data dimension D) times the number of fields $|\\Psi|$, constrained to the total budget $B$; width/depth are tuned to fit. A theorem (feasible-space comparison) shows that, at equal per-instance parameter budget, DDiF's output space strictly contains FreD's, so DDiF should reach a lower or equal distillation loss for the same instance count -- confirmed empirically by a -0.89 correlation between reconstruction error and downstream accuracy. Hard, fixed one-hot labels by default (soft-label compatibility is shown but not used in the headline tables).", "evidence": "ImageNet-Subset, 128x128, IPC=1 (#Params=491.52k, equal-parameter budget across all parameterizations): DDiF 72.0% on ImageNette vs. FreD 66.8%, SPEED 66.9%, Vanilla+RTP 69.6%, HaBa 51.9%, GLaD 38.7% (Table 1). ImageSquawk, same budget: DDiF 67.0% vs. FreD 57.0%, SPEED 60.9%. At 256x256, IPC=1: DDiF 67.8% vs. FreD 54.2%, Vanilla (full pixels) 32.1% -- DDiF beats even the uncompressed pixel baseline at higher resolution. Fixed-decoded-instance-count comparison (Table 7): at the same number of decoded instances per class (8), DDiF matches FreD's accuracy (67.1 vs 66.8 on ImageNette) using only 77.04k of the 491.52k budget FreD needs (15.7% of the budget) -- isolating coding efficiency from raw budget size. Cross-objective (Table 4): DDiF improves over baselines when paired with gradient matching (DC) and distribution matching (DM), not only trajectory matching, confirming compatibility beyond its default loss. Other modalities: video (miniUCF) at 1.7% of SDD's (a video-specific method) storage budget reaches competitive accuracy; audio (Speech Commands) and 3D voxel (ModelNet/ShapeNet) each show DDiF as best among tested parameterizations. Ablation (Fig. 7): performance improves consistently across neural-field width and depth choices and is robust to swapping the field architecture for an FFN-based one.", "limitations": "States its own limitations (Appendix D.4): less efficient on low-dimensional datasets, because a sufficiently expressive-yet-small neural field is harder to design when the output dimension is already small (the coding-efficiency advantage is shown mainly on 128x128+/multi-modal high-dimensional data, less so at CIFAR-scale). One-to-one correspondence between a synthetic instance and its neural field means there is no shared component storing cross-instance or cross-class information, unlike rtp-2022's shared memories or haba-2022's shared hallucinators/bases -- the paper suggests adding modulation/conditional codes as future work rather than solving it here. As with fred-2023, its default headline loss is trajectory matching (mtt-2022), so absolute numbers inherit that objective's expert-trajectory cost, not separately reported for DDiF's own overhead." }, "summary": "Parameterizes each synthetic instance as a neural field mapping coordinates to output quantities instead of storing pixels directly, theoretically shown to be more expressive than prior parameterizations at the same per-instance budget; reports superior performance on several benchmarks and extends beyond images to video, audio and 3D voxel data.", "abstract": "Utilizing a large-scale dataset is essential for training high-performance deep learning models, but it also comes with substantial computation and storage costs. To overcome these challenges, dataset distillation has emerged as a promising solution by compressing the large-scale dataset into a smaller synthetic dataset that retains the essential information needed for training. This paper proposes a novel parameterization framework for dataset distillation, coined Distilling Dataset into Neural Field (DDiF), which leverages the neural field to store the necessary information of the large-scale dataset. Due to the unique nature of the neural field, which takes coordinates as input and output quantity, DDiF effectively preserves the information and easily generates various shapes of data. We theoretically confirm that DDiF exhibits greater expressiveness than some previous literature when the utilized budget for a single synthetic instance is the same. Through extensive experiments, we demonstrate that DDiF achieves superior performance on several benchmark datasets, extending beyond the image domain to include video, audio, and 3D voxel. We release the code at https://github.com/aailab-kaist/DDiF.", "awesome_sections": [ "Main / Distilled Dataset Parametrization" ] }, { "id": "unidd-2026", "type": "paper", "title": "Understanding Dataset Distillation via Spectral Filtering", "short": "Spectral filtering view", "arxiv": "2503.01212", "url": null, "date": "2025-03-03", "venue": { "name": "ICLR 2026", "status": "conference" }, "authors": [ "Deyu Bo", "Songhua Liu", "Xinchao Wang" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Interprets every dataset-distillation objective as a filter function acting on the eigenvalues of the feature-feature correlation matrix and the frequencies of the feature-label correlation matrix, showing existing methods split into low-frequency (global texture) and high-frequency (local detail) matchers under one spectral-filtering framework, and proposes Curriculum Frequency Matching, which adjusts the filter during distillation to cover both regimes, outperforming baselines on CIFAR-10/100 and ImageNet-1K.", "abstract": "Dataset distillation (DD) has emerged as a promising approach to compress datasets and speed up model training. However, the underlying connections among various DD methods remain largely unexplored. In this paper, we introduce UniDD, a spectral filtering framework that unifies diverse DD objectives. UniDD interprets each DD objective as a specific filter function that affects the eigenvalues of the feature-feature correlation (FFC) matrix and modulates the frequency components of the feature-label correlation (FLC) matrix. In this way, UniDD reveals that the essence of DD fundamentally lies in matching frequency-specific features. Moreover, according to the filter behaviors, we classify existing methods into low-frequency matching and high-frequency matching, encoding global texture and local details, respectively. However, existing methods rely on fixed filter functions throughout distillation, which cannot capture the low- and high-frequency information simultaneously. To address this limitation, we further propose Curriculum Frequency Matching (CFM), which gradually adjusts the filter parameter to cover both low- and high-frequency information of the FFC and FLC matrices. Extensive experiments on small-scale datasets, such as CIFAR-10/100, and large-scale datasets, including ImageNet-1K, demonstrate the superior performance of CFM over existing baselines and validate the practicality of UniDD.", "awesome_sections": [ "Main / Better Understanding" ] }, { "id": "ncfm-2025", "type": "paper", "title": "Dataset Distillation with Neural Characteristic Function: A Minmax Perspective", "short": "NCFM", "arxiv": "2502.20653", "url": null, "date": "2025-02-28", "venue": { "name": "CVPR 2025", "status": "conference" }, "authors": [ "Shaobo Wang", "Yicun Yang", "Zhiyuan Liu", "Chenghao Sun", "Xuming Hu", "Conghui He", "Linfeng Zhang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "tier": "notable", "links": { "code": "https://github.com/gszfwsb/NCFM", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "parameterization": [ "pixels", "downsampled-multiformation" ], "labels": "mixed", "prior": [ "random-networks", "pretrained-classifier" ], "regime": "single-level", "scale": [ "mnist-cifar", "tiny-imagenet", "imagenet-subsets" ] }, "explained": { "before": "dm-2023 matches only class-wise mean embeddings (empirical MMD with a linear kernel) in random networks; idm-2023, m3d-2024 and datadam-2023 enrich the embeddings or statistics but still use a fixed, hand-chosen discrepancy. Trajectory-matching methods (mtt-2022 and successors such as DATM) remain the accuracy reference but at far higher memory cost, running out of memory on CIFAR-100 at IPC=50 on an 80GB A100.", "problem": "Fixed discrepancy measures used in prior distribution matching -- pointwise MSE or linear-kernel MMD -- only capture a first moment or a hand-picked kernel and are not sufficient conditions for the two feature distributions to actually coincide, so real and synthetic distributions can look matched under the metric while remaining different.", "idea": "Cast distribution matching as a minmax game: a lightweight sampling network learns which frequency arguments of the characteristic function make the real/synthetic discrepancy largest (an adversarially optimized, comprehensive discrepancy measure that provably determines the full distribution), while the synthetic images minimize that learned discrepancy. The characteristic-function gap decomposes into an amplitude term (distribution scale, i.e. diversity) and a phase term (distribution centers, i.e. realism).", "method": "Real and synthetic batches are embedded by a hybrid feature extractor formed by beta-blending an early and a late checkpoint (beta ~ U(0,1) each step) drawn from a pool that mixes randomly initialized and pretrained networks. A sampling network psi parameterizes a scale-mixture-of-Gaussians distribution over frequency arguments t (1024 by default); psi is trained to maximize the neural characteristic function discrepancy (NCFD) while the synthetic pixel images (with IDC-style multi-formation, scale factor rho=2) are trained to minimize it, balancing amplitude and phase terms with a hyperparameter alpha. NCFD costs linear time in the number of sampled frequencies versus MMD's quadratic time. Soft labels from a pretrained network and dataset fine-tuning are supported as optional add-ons but the paper states they are not required for the headline numbers.", "evidence": "CIFAR-10, IPC=10, ConvNet-3, hard labels: NCFM 71.8% vs. IDM 58.6%, M3D 63.5%, DM 48.9% (Table 1). CIFAR-100, IPC=1: NCFM 34.4% vs. DM 11.4% (+23.0pp) and IDM 20.1%. Tiny-ImageNet, IPC=50: NCFM 29.6% vs. IDM 27.7%. ImageSquawk (128x128), IPC=10: NCFM 72.8% vs. MTT 52.3% (+20.5pp, Table 2). Cross-architecture (CIFAR-10, IPC=50, trained on ConvNet): NCFM 77.4/75.5/75.5/73.8% on ConvNet/AlexNet/VGG/ResNet vs. DM 65.2/61.3/59.9/57.0% (Table 4). Cost (Table 3, CIFAR-100, IPC=50, A100): NCFM 1.36s/iter and 7.22GB vs. TESLA 28.24s/OOM and MTT/FTD/DATM all OOM -- roughly 20x faster and >300x less memory than the trajectory-matching baselines the abstract cites. Ablation (Table 5): removing the sampling network psi (holding data curation fixed) costs 3.2pp on CIFAR-10 IPC=50 (74.2%->77.4%) and 10.1pp on Tiny-ImageNet IPC=10 (14.2%->24.3%), isolating the adversarial frequency-sampling mechanism from the CFD metric itself.", "limitations": "The ablation section states explicitly that its \"no sampling network\" comparison uses no fine-tuning or soft-label integration, implying the headline Table 1/2/4 numbers do include those data-curation add-ons by default; since the DM/IDM/M3D/MTT baselines it is compared against in those tables report hard-label, no-curation numbers from their own papers, part of the gap may be attributable to this protocol difference rather than the NCFD metric alone, and the paper does not report a hard-label-only, no-curation version of its headline table to isolate this. The hybrid random+pretrained feature extractor and the fine-tuning/ soft-label option both introduce a pretrained-teacher dependency; the paper does not quantify its cost. No ImageNet-1K full-scale result is reported (only 128x128 ImageNet subsets and low-resolution benchmarks)." }, "summary": "Reformulates distribution matching as a minmax problem using a Neural Characteristic Function Discrepancy, where a network optimizes the sampling of characteristic-function frequency arguments to maximize the measured discrepancy while the synthetic data is optimized to minimize it, aligning phase and amplitude of real and synthetic features in the complex plane; reports a 20.5-point gain on ImageSquawk, over 300x less GPU memory, and the first lossless CIFAR-100 compression on a single 2080 Ti.", "abstract": "Dataset distillation has emerged as a powerful approach for reducing data requirements in deep learning. Among various methods, distribution matching-based approaches stand out for their balance of computational efficiency and strong performance. However, existing distance metrics used in distribution matching often fail to accurately capture distributional differences, leading to unreliable measures of discrepancy. In this paper, we reformulate dataset distillation as a minmax optimization problem and introduce Neural Characteristic Function Discrepancy (NCFD), a comprehensive and theoretically grounded metric for measuring distributional differences. NCFD leverages the Characteristic Function (CF) to encapsulate full distributional information, employing a neural network to optimize the sampling strategy for the CF's frequency arguments, thereby maximizing the discrepancy to enhance distance estimation. Simultaneously, we minimize the difference between real and synthetic data under this optimized NCFD measure. Our approach, termed Neural Characteristic Function Matching (\\mymethod{}), inherently aligns the phase and amplitude of neural features in the complex plane for both real and synthetic data, achieving a balance between realism and diversity in synthetic samples. Experiments demonstrate that our method achieves significant performance gains over state-of-the-art methods on both low- and high-resolution datasets. Notably, we achieve a 20.5\\% accuracy boost on ImageSquawk. Our method also reduces GPU memory usage by over 300$\\times$ and achieves 20$\\times$ faster processing speeds compared to state-of-the-art methods. To the best of our knowledge, this is the first work to achieve lossless compression of CIFAR-100 on a single NVIDIA 2080 Ti GPU using only 2.3 GB of memory.", "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ], "builds_on": [ "dm-2023", "idm-2023", "m3d-2024", "idc-2022" ] }, { "id": "grad-mm-2025", "type": "paper", "title": "Synthetic Text Generation for Training Large Language Models via Gradient Matching", "short": "GRADMM", "arxiv": "2502.17607", "url": null, "date": "2025-02-24", "venue": { "name": "ICML 2025", "status": "conference" }, "authors": [ "Dang Nguyen", "Zeman Li", "Mohammadhossein Bateni", "Vahab Mirrokni", "Meisam Razaviyayn", "Baharan Mirzasoleiman" ], "kind": "method", "settings": [ "text" ], "families": [ "gradient-matching" ], "builds_on": [ "dilm-2024", "sldd-2021" ], "tier": "notable", "links": { "code": "https://github.com/BigML-CS-UCLA/GRADMM", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Proposes GRADMM, which uses ADMM to optimize the embeddings of synthetic text examples so their gradient matches the (noisy) gradient of real training or validation data, then decodes the embeddings into low-perplexity token sequences, giving convergence, performance and privacy guarantees for fine-tuning LLMs on human-readable synthetic text; reports effectiveness across several classification tasks.", "abstract": "Synthetic data has the potential to improve the performance, training efficiency, and privacy of real training examples. Nevertheless, existing approaches for synthetic text generation are mostly heuristics and cannot generate human-readable text without compromising the privacy of real data, or provide performance guarantees for training Large Language Models (LLMs). In this work, we propose the first theoretically rigorous approach for generating synthetic human-readable text that provides convergence, performance, and privacy guarantees for fine-tuning LLMs on a target task. To do so, we leverage Alternating Direction Method of Multipliers (ADMM) that iteratively optimizes the embeddings of synthetic examples to match the noisy gradient of the target training or validation data, and maps them to a sequence of text tokens with low perplexity. In doing so, the generated synthetic text guarantees convergence of the model to a close neighborhood of the solution obtained by fine-tuning on real data and preserves their privacy. Experiments on various classification tasks confirm the effectiveness of our proposed approach. Our code is available at https://github.com/BigML-CS-UCLA/GRADMM.", "awesome_sections": [ "Applications / Text" ], "notes": "", "explained": { "before": "Embedding-level text distillation (sldd-2021/TDD and successors) optimizes continuous word embeddings that are unreadable and tied to one embedding table; dilm-2024's DiLM fixes readability by training a generator LM with gradient matching, but that requires training a proxy LM from scratch (a scaling bottleneck) and, because the generated text can include memorized real examples, provides no privacy guarantee; separately, LLM-prompted synthetic data generation (zero-/few-shot GPT-4-style pipelines) is heuristic, offers no convergence guarantee for the downstream model, and does not preserve privacy of the source data either.", "problem": "No existing method for generating synthetic text for LLM fine-tuning is simultaneously human-readable, scalable to LLMs with billions of parameters, and accompanied by a formal convergence, performance, or privacy guarantee; text's discreteness makes the underlying optimization problem (find a low-perplexity token sequence whose gradient matches real data) NP-hard in principle.", "idea": "Reformulate discrete-text gradient matching as a constrained non-convex optimization — synthetic token embeddings must both lie in the model's vocabulary embedding set and satisfy a perplexity bound — and solve it with ADMM, alternating between a continuous gradient-matching update and a projection back onto (readable, low-perplexity) vocabulary tokens, which yields provable convergence and, by privatizing the target gradient before matching, provable differential privacy.", "method": "Given a pretrained LLM (Phi) and a target dataset $\\mathcal{D}_{real}$, GRADMM introduces an auxiliary variable $\\mathbf{Z}$ constrained to the vocabulary embedding set $\\mathcal{E}$ and solves $\\min_\\mathbf{X} f(\\mathbf{X}) + \\mathcal{I}_\\mathcal{E}(\\mathbf{Z})$ s.t. $\\mathbf{X}=\\mathbf{Z}$ via ADMM: a primal step optimizes synthetic embeddings $\\mathbf{X}$ (with Adam) to match a clipped, Gaussian-noised (for $(\\varepsilon,\\delta)$-DP) real-data gradient at the pretrained parameters (last-layer gradient only, for tractability and speed — 2.6x less memory, 2.3x less time than full- gradient matching); a projection step maps embeddings to the top-$k$ most probable vocabulary tokens conditioned on previously decoded tokens (top-$k$ decoding, not nearest-neighbor, to keep text coherent); and a dual step updates Lagrange multipliers. A post-hoc filtering stage removes synthetic examples with the wrong predicted label (via few-shot evaluation), keeps only the lowest-gradient-matching-loss examples per class, and rebalances per-class average loss. The convergence proof (Lemma 4.1, Theorem 4.2, Corollary 4.3) bounds the fine-tuned model's parameter/loss distance from a real-data fine-tuned model as a function of the residual gradient-matching error, under smoothness and PL*-condition assumptions appropriate to short fine-tuning runs. Nothing beyond the synthetic text itself is stored; no generator network is kept (unlike DiLM).", "evidence": "Table 1 (SST-2/Tweet-emotions/Rotten-tomatoes, Phi model, hard labels, various data sizes): GRADMM ($\\varepsilon=\\infty$) beats zero-shot LLM generation, few-shot LLM generation, and Herding/K-center/ Random real-data selection by up to 13.1% (e.g. SST-2 n=5: 86.5% vs. best baseline 75.3% Random), and the DP version ($\\varepsilon=0.05$) only slightly trails the non-DP version (e.g. SST-2 n=5: 84.2% vs. 86.5%) while being the only method with a privacy guarantee. Data-scarce regime (Figure 1): generating 100 synthetic examples from just 5 real validation examples outperforms training directly on those 5 real examples by 15.7-31.5% across three datasets. Table 2 (cross-model transfer): text generated by matching Phi's gradient, when used to fine-tune Llama-3.2-1B and OPT-1.3B instead, still beats zero- shot/few-shot/coreset baselines in most cells (e.g. Llama-3.2-1B SST-2: 89.4% vs. 88.4% best baseline). Privacy: loss-based membership-inference-attack advantage is -2.5% to -2.9% (near chance) for both DP and non-DP variants (Sec. 5.3). Cost: matching only the last-layer gradient (vs. full gradient) cuts generation memory from 44.6GB to 17.3GB and generation time from 4.6 to 2 hours on one H100 GPU (Table 4).", "limitations": "Requires access to per-sample real-data gradients from the target LLM, so it needs white-box access to a (possibly large) model rather than only its outputs; the top-$k$ projection/filtering pipeline means synthetic examples are generated independently rather than jointly optimized, so the final filtered set only approximately matches the target gradient; experiments use one mid-sized base generator model (Phi) and three short-text classification datasets, not open-ended generation, larger sequence lengths, or full LM pretraining; the convergence guarantees rely on fine-tuning-specific smoothness/PL* assumptions that may not hold for longer training runs or pretraining from scratch." } }, { "id": "survey-liu-2025", "type": "paper", "title": "The Evolution of Dataset Distillation: Toward Scalable and Generalizable Solutions", "short": "Liu & Du survey", "arxiv": "2502.05673", "url": null, "date": "2025-02-08", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Ping Liu", "Jiawei Du" ], "kind": "survey", "settings": [ "image-classification" ], "families": [], "tier": "core", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "This survey focuses on post-2023 advances that scale dataset distillation to ImageNet-1K and ImageNet-21K, organizing progress into trajectory matching, gradient matching, distribution matching, scalable generative approaches and decoupled optimization, and highlights SRe2L, soft-label strategies and lossless-distillation techniques as the field's key breakthroughs. It also reviews robustness to adversarial and backdoor attacks, non-IID data, and emerging applications in video, audio, multimodal learning, medical imaging and scientific computing.", "abstract": "Dataset distillation, which condenses large-scale datasets into compact synthetic representations, has emerged as a critical solution for training modern deep learning models efficiently. While prior surveys focus on developments before 2023, this work comprehensively reviews recent advances, emphasizing scalability to large-scale datasets such as ImageNet-1K and ImageNet-21K. We categorize progress into a few key methodologies: trajectory matching, gradient matching, distribution matching, scalable generative approaches, and decoupling optimization mechanisms. As a comprehensive examination of recent dataset distillation advances, this survey highlights breakthrough innovations: the SRe2L framework for efficient and effective condensation, soft label strategies that significantly enhance model accuracy, and lossless distillation techniques that maximize compression while maintaining performance. Beyond these methodological advancements, we address critical challenges, including robustness against adversarial and backdoor attacks, effective handling of non-IID data distributions. Additionally, we explore emerging applications in video and audio processing, multi-modal learning, medical imaging, and scientific computing, highlighting its domain versatility. By offering extensive performance comparisons and actionable research directions, this survey equips researchers and practitioners with practical insights to advance efficient and generalizable dataset distillation, paving the way for future innovations.", "notes": "" }, { "id": "dark-distillation-backdooring-distilled-2026", "type": "paper", "title": "Dark Distillation: Backdooring Distilled Datasets without Accessing Raw Data", "arxiv": "2502.04229", "url": null, "date": "2025-02-06", "venue": { "name": "AAAI 2026", "status": "conference" }, "authors": [ "Ziyuan Yang", "Ming Yan", "Yi Zhang", "Joey Tianyi Zhou" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [], "applications": [ "backdoor" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Shows that an attacker who intercepts an already-distilled dataset in transit, with no access to the raw data it came from, can still inject a working backdoor by reconstructing per-class conceptual archetypes from a model trained on the distilled set and updating those archetypes with a hybrid loss that adds backdoor information while preserving the original optimization trajectory. Reports that distilled datasets remain highly vulnerable to this raw-data-free attack across multiple raw datasets and distillation methods.", "abstract": "Dataset distillation (DD) enhances training efficiency and reduces bandwidth by condensing large datasets into smaller synthetic ones. It enables models to achieve performance comparable to those trained on the raw full dataset and has become a widely adopted method for data sharing. However, security concerns in DD remain underexplored. Existing studies typically assume that malicious behavior originates from dataset owners during the initial distillation process, where backdoors are injected into raw datasets. In contrast, this work is the first to address a more realistic and concerning threat: attackers may intercept the dataset distribution process, inject backdoors into the distilled datasets, and redistribute them to users. While distilled datasets were previously considered resistant to backdoor attacks, we demonstrate that they remain vulnerable to such attacks. Furthermore, we show that attackers do not even require access to any raw data to inject the backdoors successfully. Specifically, our approach reconstructs conceptual archetypes for each class from the model trained on the distilled dataset. Backdoors are then injected into these archetypes to update the distilled dataset. Moreover, we ensure the updated dataset not only retains the backdoor but also preserves the original optimization trajectory, thus maintaining the knowledge of the raw dataset. To achieve this, a hybrid loss is designed to integrate backdoor information along the benign optimization trajectory, ensuring that previously learned information is not forgotten. Extensive experiments demonstrate that distilled datasets are highly vulnerable to backdoor attacks, with risks pervasive across various raw datasets, distillation methods, and downstream training strategies. Moreover, our attack method is efficient, capable of synthesizing a malicious distilled dataset in under one minute in certain cases.", "awesome_sections": [ "Applications / Privacy" ], "notes": "families left empty: the attack targets an already-distilled dataset regardless of which mechanism family produced it." }, { "id": "td3-2025", "type": "paper", "title": "TD3: Tucker Decomposition Based Dataset Distillation Method for Sequential Recommendation", "short": "TD3", "arxiv": "2502.02854", "url": null, "date": "2025-02-05", "venue": { "name": "WWW 2025", "status": "conference" }, "authors": [ "Jiaqing Zhang", "Mingjia Yin", "Hao Wang", "Yawen Li", "Yuyang Ye", "Xingyu Lou", "Junping Du", "Enhong Chen" ], "kind": "method", "settings": [ "other-data" ], "families": [ "meta-learning-bptt", "parameterization" ], "facets": { "parameterization": [ "factorized-bases" ] }, "tier": "catalogued", "links": { "code": "https://github.com/USTC-StarTeam/TD3", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "TD3 distills sequential-recommendation interaction data within a meta-learning bi-level framework, using Tucker decomposition to store the synthetic sequence summary as four compact factors (synthetic user latent factor, temporal dynamics factor, shared item latent factor, and a relation core) instead of a full interaction tensor, and adds a surrogate feature-space alignment objective on top of naive performance matching, optimized with RaT-BPTT. The paper reports superior and cross-architecture generalizable performance versus prior baselines on multiple public recommendation datasets.", "abstract": "In the era of data-centric AI, the focus of recommender systems has shifted from model-centric innovations to data-centric approaches. The success of modern AI models is built on large-scale datasets, but this also results in significant training costs. Dataset distillation has emerged as a key solution, condensing large datasets to accelerate model training while preserving model performance. However, condensing discrete and sequentially correlated user-item interactions, particularly with extensive item sets, presents considerable challenges. This paper introduces \\textbf{TD3}, a novel \\textbf{T}ucker \\textbf{D}ecomposition based \\textbf{D}ataset \\textbf{D}istillation method within a meta-learning framework, designed for sequential recommendation. TD3 distills a fully expressive \\emph{synthetic sequence summary} from original data. To efficiently reduce computational complexity and extract refined latent patterns, Tucker decomposition decouples the summary into four factors: \\emph{synthetic user latent factor}, \\emph{temporal dynamics latent factor}, \\emph{shared item latent factor}, and a \\emph{relation core} that models their interconnections. Additionally, a surrogate objective in bi-level optimization is proposed to align feature spaces extracted from models trained on both original data and synthetic sequence summary beyond the na\\\"ive performance matching approach. In the \\emph{inner-loop}, an augmentation technique allows the learner to closely fit the synthetic summary, ensuring an accurate update of it in the \\emph{outer-loop}. To accelerate the optimization process and address long dependencies, RaT-BPTT is employed for bi-level optimization. Experiments and analyses on multiple public datasets have confirmed the superiority and cross-architecture generalizability of the proposed designs. Codes are released at https://github.com/USTC-StarTeam/TD3.", "builds_on": [ "rat-bptt-2024" ], "notes": "" }, { "id": "on-learning-representations-for-2025", "type": "paper", "title": "On Learning Representations for Tabular Data Distillation", "short": "TDColER", "arxiv": "2501.13905", "url": null, "date": "2025-01-23", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Inwon Kang", "Parikshit Ram", "Yi Zhou", "Horst Samulowitz", "Oshani Seneviratne" ], "kind": "method", "settings": [ "other-data" ], "families": [], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Studies representation learning for tabular dataset distillation, where feature heterogeneity and non-differentiable learners (tree ensembles, nearest-neighbor predictors) break image-style pipelines, and proposes TDColER, a column-embedding representation-learning framework layered on top of existing distillation schemes, evaluated on TDBench, a new benchmark of 226,890 distilled tabular datasets and 548,880 downstream models. Reports TDColER improving distilled-data quality by 0.5-143% across seven tabular learning models.", "abstract": "Dataset distillation generates a small set of information-rich instances from a large dataset, resulting in reduced storage requirements, privacy or copyright risks, and computational costs for downstream modeling, though much of the research has focused on the image data modality. We study tabular data distillation, which brings in novel challenges such as the inherent feature heterogeneity and the common use of non-differentiable learning models (such as decision tree ensembles and nearest-neighbor predictors). To mitigate these challenges, we present $\\texttt{TDColER}$, a tabular data distillation framework via column embeddings-based representation learning. To evaluate this framework, we also present a tabular data distillation benchmark, ${{\\sf \\small TDBench}}$. Based on an elaborate evaluation on ${{\\sf \\small TDBench}}$, resulting in 226,890 distilled datasets and 548,880 models trained on them, we demonstrate that $\\texttt{TDColER}$ is able to boost the distilled data quality of off-the-shelf distillation schemes by 0.5-143% across 7 different tabular learning models.", "notes": "Boosts existing distillation schemes via representation learning rather than proposing a new matching mechanism itself; families left empty." }, { "id": "pushforward-quantization-2026", "type": "paper", "title": "Dataset Distillation as Pushforward Optimal Quantization", "short": "Pushforward quantization", "arxiv": "2501.07681", "url": null, "date": "2025-01-13", "venue": { "name": "ICLR 2026", "status": "conference" }, "authors": [ "Hong Ye Tan", "Emma Slade" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Shows that decoupled/disentangled dataset-distillation methods with an encoder-decoder structure are instances of classical optimal quantization and Wasserstein-barycenter problems, and uses this connection to propose clustering in a diffusion model's latent space to select prototype points, reporting better performance and cross-model generalization than D4M on ImageNet-1K, plus state-of-the-art results using the distilled noise initializations with a stronger diffusion transformer.", "abstract": "Dataset distillation aims to find a synthetic training set such that training on the synthetic data achieves similar performance to training on real data, with orders of magnitude less computational requirements. Existing methods can be broadly categorized as either bi-level optimization problems that have neural network training heuristics as the lower level problem, or disentangled methods that bypass the bi-level optimization by matching distributions of data. The latter method has the major advantages of speed and scalability in terms of size of both training and distilled datasets. We demonstrate that when equipped with an encoder-decoder structure, the empirically successful disentangled methods can be reformulated as an optimal quantization problem, where a finite set of points is found to approximate the underlying probability measure by minimizing the expected projection distance. In particular, we link existing disentangled dataset distillation methods to the classical optimal quantization and Wasserstein barycenter problems, demonstrating consistency of distilled datasets for diffusion-based generative priors. We propose Dataset Distillation by Optimal Quantization, based on clustering in a latent space. Compared to the previous SOTA method D\\textsuperscript{4}M, we achieve better performance and inter-model generalization on the ImageNet-1K dataset with trivial additional computation, and SOTA performance in higher image-per-class settings. Using the distilled noise initializations in a stronger diffusion transformer model, we obtain SOTA distillation performance on ImageNet-1K and its subsets, outperforming diffusion guidance methods.", "awesome_sections": [ "Main / Better Optimization" ], "notes": "Family corrected from the draft's optimization-recipes guess to diffusion-guided-synthesis: the method operates on diffusion-model latents and is compared directly against D4M — flagged for review.", "facets": { "prior": [ "diffusion" ], "regime": "training-free", "parameterization": [ "pixels" ], "scale": [ "imagenet-subsets", "imagenet-1k" ] }, "builds_on": [ "d4m-2024", "minimax-2024", "igd-2025" ], "explained": { "before": "D4M clusters a target dataset's latents into per-class prototypes and decodes them through a pretrained latent diffusion model with no theoretical account of why clustering should approximate the data distribution well; IGD and Minimax (used later on a DiT backbone) add guidance or fine-tuning without a consistency argument either.", "problem": "No prior dataset-distillation work proves that a distilled set (produced by clustering-and-decoding) is a provably good approximation of the population risk gradient as the budget grows -- the field lacks a theoretical account of why disentangled, encoder-decoder-based methods like D4M work, or what governs their error.", "idea": "Disentangled distillation methods with an encoder-decoder structure are literally instances of optimal quantization / Wasserstein-barycenter problems: choosing K prototype points per class to minimize expected projection distance to the class's latent distribution. Framed this way, (a) a proper optimal-quantization clustering (weighted mini-batch k-means with per-cluster population weights, not unweighted k-means or a Wasserstein-barycenter objective) provably minimizes distortion and gives convergent gradient-risk approximations as more points are added, and (b) diffusing the population forward and clustering in the (higher-fidelity) diffusion latent space at a partial noise level preserves this guarantee via a Wasserstein-continuity bound on diffusion SDEs.", "method": "A four-step pipeline: (1) encode the real per-class training data into a pretrained diffusion model's latent space (LDM/UNet in the main experiments, DiT in a secondary set); (2) run mini-batch k-means (equivalent to a competitive-learning vector-quantization, CLVQ, algorithm) per class in that latent space, recording each cluster's population count $v_k$; (3) decode the resulting cluster centers through the pretrained (frozen, untrained/unfine-tuned) diffusion decoder to get the distilled images; (4) train the student with a population-count-derived per-sample weight $w_k = v_k/\\sum_j v_j$ folded into a weighted cross-entropy loss, $\\min_\\theta \\sum_{(x,y,w)} w\\cdot\\ell(x,y,\\theta)$, rather than treating every distilled sample as equally likely. Theorem 1 bounds the difference in expectation of any Lipschitz function between two latent distributions diffused backward to a partial time $\\delta$ by their Wasserstein-2 distance at $t=T$, times a Lipschitz constant, justifying clustering at a partial (not full) noise level; Corollary 1 gives an $O(K^{-1/d})$ convergence rate in quantization points $K$ and latent dimension $d$.", "evidence": "ImageNet-1K, soft-label (ResNet-18 teacher, ceiling ~69.8), Table 2: IPC10, ResNet-18/50/101: 33.1+/-0.60 / 34.4+/-0.99 / 36.7+/-0.80 vs D4M 27.9 / 33.5 / 34.2 (numbers for D4M taken from the original paper); IPC50: 56.2+/-0.07 / 62.5+/-0.24 / 63.6+/-0.13 vs D4M 55.2 / 62.4 / 63.4; IPC200, ResNet-101: 68.6+/-0.08 vs D4M 68.1, a \"30% reduction in error gap\" at IPC200/ResNet-101. On a stronger DiT backbone (Table 4): ImageNet-1K IPC10/50: 53.0+/-0.2 / 62.7+/-0.1 vs IGD 45.5+/-0.5 / 59.8+/-0.3 and Minimax-IGD 46.2+/-0.6 / 60.3+/-0.4; ImageWoof IPC10/50: 48.8+/-2.0 / 65.4+/-0.7 vs IGD 41.0+/-0.8 / 62.7+/-1.2, Minimax-IGD 43.3+/-0.3 / 65.0+/-0.8. No MGD3, D3HR or CaO2 comparison anywhere. Wasserstein validation (Table 1): the population-weighted clustering reduces latent W2 distance to the target vs D4M's unweighted clustering by 15.7% at IPC10 and 16.1% at IPC50 (class 0 example: 49.67 to 42.97 at IPC10). Cross-architecture generalization (Table 3, ResNet-18 teacher): uniformly beats D4M on ResNet-18/MobileNet-V2/EfficientNet-B0 students, slightly underperforms D4M with a Swin-T student. Cost: \"trivial additional computation\" over D4M (Appendix G); memory constant in IPC, matching D4M's advantage over bi-level methods.", "limitations": "Stated: k-means/Lloyd's-algorithm clustering can converge to a stationary but not globally optimal grid in higher dimensions; the consistency proof covers this specific quantization-based construction only, explicitly leaving \"consistency or convergence of other dataset distillation frameworks such as bi-level methods and diffusion guidance\" as open questions the paper does not resolve; the DiT-backbone gains are empirical, and the paper notes that sharper bounds exploiting sub-Gaussianity remain future work. Observed: no MGD3, D3HR or CaO2 comparison, so it is untested against the family's mode-guidance and inversion-latent-statistics branches; slightly underperforms D4M for transformer-architecture students (Swin-T), suggesting the quantization-optimality argument does not uniformly transfer across evaluator architectures; baseline D4M numbers on the UNet backbone are quoted from the original paper rather than re-run under the paper's own protocol." } }, { "id": "cv-dd-2025", "type": "paper", "title": "Dataset Distillation via Committee Voting", "short": "CV-DD", "arxiv": "2501.07575", "url": null, "date": "2025-01-13", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Jiacheng Cui", "Zhaoyi Li", "Xiaochen Ma", "Xinyue Bi", "Yaxin Luo", "Zhiqiang Shen" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "facets": { "labels": "soft-relabel", "prior": [ "pretrained-classifier" ], "regime": "decoupled", "parameterization": [ "pixels" ], "scale": [ "imagenet-1k" ] }, "tier": "catalogued", "links": { "code": "https://github.com/Jiacheng8/CV-DD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "SRe2L inverts a single frozen teacher and stores its soft labels; G-VBSM and EDC extend this to several backbones but weight every committee member equally when forming the recovery loss and the relabeling logits. CDA improves the augmentation curriculum and RDED replaces inversion with real-crop selection, but neither addresses multi-model weighting.", "problem": "Single-backbone synthesis (SRe2L) restricts diversity and bakes in one model's bias; equal-weight multi-backbone synthesis (G-VBSM) assumes every pretrained model is equally informative, which the paper argues is false and dilutes the distilled set's quality; and post hoc soft labels computed with the teacher's real-data running batch-norm statistics are shown (Fig. 5) to mismatch the actual batch statistics of synthetic images, producing suboptimal supervision.", "idea": "Weight a committee of pretrained backbones by each one's own measured \"prior performance\" (its distill-then-train-a-student accuracy) rather than uniformly, both when forming the recovery gradient and when generating soft labels, and recompute batch-norm statistics from each synthetic batch itself (not from the teacher's stored real-data running statistics) before producing the labels used to train students.", "method": "Builds SRe2L++ first: real-image initialization (instead of Gaussian noise), RandomResizedCrop augmentation during recovery, a smoothed cosine learning-rate schedule and smaller evaluation batch size. On top of this, Algorithm 1 pretrains each committee model $\\Phi$, distills a set with it alone, trains a student on that set with KD, and records the student's held-out accuracy as $\\alpha_\\Phi$ (\"prior performance\"). During synthesis, a random subset of $N{=}2$ committee members is sampled per step and their per-model losses are combined with a softmax-over-$\\alpha_\\Phi$ weighting (temperature $T{=}5$, Eq. 4) instead of equal averaging; the same prior-weighted softmax combines committee predictions into soft labels. Batch-Specific Soft Labeling (BSSL) recomputes batch-norm mean/variance (Eqs. 6-7) from the current synthetic batch's own activations, holding the rest of the teacher fixed, before scoring soft labels; for non-BN architectures (ViT) a BN-ViT variant substitutes BatchNorm for LayerNorm so BSSL still applies. No student network is in the synthesis loop; five backbones (ResNet-18/50, DenseNet-121, ShuffleNetV2, MobileNetV2) form the default committee for ImageNet-1K/Tiny-ImageNet.", "evidence": "ImageNet-1K, ResNet-18, relabel+KD, 300 evaluation epochs (Table 1): IPC=10, CV-DD 49.5% vs its own SRe2L++ baseline 43.1%, RDED 42.0%, CDA 33.6%; IPC=50, CV-DD 59.5% vs SRe2L++ 57.6%, RDED 56.5%, CDA 53.5%. Against vanilla-ensemble methods under a matched protocol (Table 2, ResNet-18): IPC=10, CV-DD 49.5% vs EDC 48.6%, G-VBSM 31.5%; IPC=50, CV-DD 59.5% vs EDC 58.0%. CIFAR-100 IPC=10: CV-DD 61.8% vs SRe2L++ 56.7%, RDED 42.6%, CDA 49.8%. Cross-architecture at IPC=10 (Table 6, nine backbones): CV-DD leads on every one, e.g. ResNet-101 57.2% vs EDC 51.7%, RegNetX-8GF 60.9% vs SRe2L++ 53.4%. Cost: committee \"prior evaluation\" (pretraining + distillation + evaluation per backbone) totals 84.7 h on ImageNet-1K (Table 9a, one RTX-4090 per model, e.g. ResNet-50: 14.4 h pretrain + 4.4 h generation + 1.5 h eval, 23.0 GB peak); full IPC=50 distillation including that prior-evaluation cost is 137.5 h vs a re-estimated 187.5 h for G-VBSM (Table 9b); per-image per-iteration recovery cost is 1.91 ms for CV-DD vs 4.32 ms (G-VBSM) and 4.99 ms (EDC) on an RTX-4090 (Table 10). Ablation: BSSL alone adds +7.0 points to CV-DD at ImageNet-1K IPC=10 (42.5% to 49.5%, Table 3e) and +4.6 to SRe2L++ (38.5% to 43.1%); prior-weighted voting over equal-weighted voting adds a further +1.1 (60.7% to 61.8%, CIFAR-100 IPC=10, Table 3b) and over random weighting +2.0; committee size scaling from one to five backbones raises ImageNet-1K IPC=10 accuracy from 43.1% to 49.5% monotonically (Table 3d). CV-DD as a plug-in on top of RDED (Table 5, ImageNet-1K IPC=10) raises 42.0% to 44.8% with prior voting vs 43.2% with equal voting. No hard-label result is reported anywhere in the paper.", "limitations": "No hard-label evaluation is reported, so the accuracy gains cannot be separated from the paper's own soft-label/relabeling machinery. The committee's \"prior performance\" scores must be computed once per backbone via a full distill-train-evaluate cycle (Algorithm 1) before any synthesis can begin, an upfront cost (84.7 h on ImageNet-1K, Table 9a) the headline efficiency comparison folds into total time but that is not amortizable across datasets. $N{>}2$ experts per step actively hurts accuracy (Table 3c), so the diversity benefit of the committee is capped by how many models can be combined per step, not by committee size $|S|$ itself. Soft-label storage size is not reported. The conclusion flags a general risk: committee members inherit and can propagate biases of the pretrained models used to build the committee." }, "summary": "Aggregates distributions and predictions from a committee of multiple pretrained models to produce higher-quality soft labels and reduce model-specific bias in decoupled synthesis; reports consistent gains over single- and multi-model distillation baselines and improved generalization to non-training-based frameworks and synthetic-to-real transfer tasks.", "abstract": "Dataset distillation aims to synthesize a compact yet representative dataset that preserves the essential characteristics of the original data for efficient model training. Existing methods mainly focus on improving data-synthetic alignment or scaling distillation to large datasets. In this work, we propose $\\textbf{C}$ommittee $\\textbf{V}$oting for $\\textbf{D}$ataset $\\textbf{D}$istillation ($\\textbf{CV-DD}$), an orthogonal approach that leverages the collective knowledge of multiple models to produce higher-quality distilled data. We first establish a strong baseline that achieves state-of-the-art performance through modern architectural and optimization choices. By integrating distributions and predictions from multiple models and generating high-quality soft labels, our method captures a broader range of data characteristics, reduces model-specific bias and the impact of distribution shifts, and significantly improves generalization. This voting-based strategy enhances diversity and robustness, alleviates overfitting, and improves post-evaluation performance. Extensive experiments across multiple datasets and IPC settings demonstrate that CV-DD consistently outperforms single- and multi-model distillation methods and generalizes well to non-training-based frameworks and challenging synthetic-to-real transfer tasks. Code is available at: https://github.com/Jiacheng8/CV-DD.", "builds_on": [ "sre2l-2023", "g-vbsm-2024", "edc-2024", "cda-2024", "rded-2024" ], "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "focusdd-2025", "type": "paper", "title": "FocusDD: Real-World Scene Infusion for Robust Dataset Distillation", "short": "FocusDD", "arxiv": "2501.06405", "url": null, "date": "2025-01-11", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Youbing Hu", "Yun Cheng", "Olga Saukh", "Firat Ozdemir", "Anqi Lu", "Zhiqiang Cao", "Zhijun Li" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Uses a pretrained Vision Transformer to locate key information patches in real images and stitches several such multi-object patches into one resolution-independent distilled image, augmented with a downsampled view of the original for generalization; the first method to apply distilled datasets to object detection as well as classification, reporting 71.0%/62.6% ImageNet-1K validation accuracy for ResNet50/MobileNet-v2 at IPC 100, and 24.4%/32.1% mAP for YOLOv11n/s on COCO2017 at IPC 50.", "abstract": "Dataset distillation has emerged as a strategy to compress real-world datasets for efficient training. However, it struggles with large-scale and high-resolution datasets, limiting its practicality. This paper introduces a novel resolution-independent dataset distillation method Focus ed Dataset Distillation (FocusDD), which achieves diversity and realism in distilled data by identifying key information patches, thereby ensuring the generalization capability of the distilled dataset across different network architectures. Specifically, FocusDD leverages a pre-trained Vision Transformer (ViT) to extract key image patches, which are then synthesized into a single distilled image. These distilled images, which capture multiple targets, are suitable not only for classification tasks but also for dense tasks such as object detection. To further improve the generalization of the distilled dataset, each synthesized image is augmented with a downsampled view of the original image. Experimental results on the ImageNet-1K dataset demonstrate that, with 100 images per class (IPC), ResNet50 and MobileNet-v2 achieve validation accuracies of 71.0% and 62.6%, respectively, outperforming state-of-the-art methods by 2.8% and 4.7%. Notably, FocusDD is the first method to use distilled datasets for object detection tasks. On the COCO2017 dataset, with an IPC of 50, YOLOv11n and YOLOv11s achieve 24.4% and 32.1% mAP, respectively, further validating the effectiveness of our approach.", "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "generative-dataset-distillation-based-2025", "type": "paper", "title": "Generative Dataset Distillation Based on Self-knowledge Distillation", "arxiv": "2501.04202", "url": null, "date": "2025-01-08", "venue": { "name": "ICASSP 2025", "status": "conference" }, "authors": [ "Longzhen Li", "Guang Li", "Ren Togo", "Keisuke Maeda", "Takahiro Ogawa", "Miki Haseyama" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "generator-as-dataset" ], "facets": { "parameterization": [ "generator-weights" ], "regime": "generator-fine-tuning" }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Improves generative dataset distillation (a follow-up to the same authors' global/local-balancing GAN method) by integrating self-knowledge distillation for more precise logit-based distribution matching between synthetic and original data, adding a standardization step on logits before matching to keep their range consistent; reports gains over existing generative distillation baselines.", "abstract": "Dataset distillation is an effective technique for reducing the cost and complexity of model training while maintaining performance by compressing large datasets into smaller, more efficient versions. In this paper, we present a novel generative dataset distillation method that can improve the accuracy of aligning prediction logits. Our approach integrates self-knowledge distillation to achieve more precise distribution matching between the synthetic and original data, thereby capturing the overall structure and relationships within the data. To further improve the accuracy of alignment, we introduce a standardization step on the logits before performing distribution matching, ensuring consistency in the range of logits. Through extensive experiments, we demonstrate that our method outperforms existing state-of-the-art methods, resulting in superior distillation performance.", "builds_on": [ "generative-dataset-distillation-balancing-2024" ], "awesome_sections": [ "Main / Generative Distillation / GAN" ], "notes": "Reclassified from generative-latent-optimization to generator-as-dataset, consistent with the closely related generative-dataset-distillation-balancing-2024 by an overlapping author team." }, { "id": "rome-is-forged-in-2025", "type": "paper", "title": "ROME is Forged in Adversity: Robust Distilled Datasets via Information Bottleneck", "short": "ROME", "arxiv": null, "url": "https://openreview.net/forum?id=agtwOsnLUB", "date": "2025-01-01", "venue": { "name": "ICML 2025", "status": "conference" }, "authors": [ "Zheng Zhou" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "applications": [ "adversarial-robustness" ], "tier": "notable", "links": { "code": "https://github.com/zhouzhengqd/ROME", "project": "https://zhouzhengqd.github.io/rome.page/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "openreview", "depth": "abstract" }, "summary": "Applies the Information Bottleneck principle to distillation robustness with two loss terms — a performance-aligned term to preserve clean accuracy and a robustness-aligned term that aligns feature distributions between synthetic and adversarially perturbed images — and introduces an Improved Robustness Ratio (I-RR) metric to evaluate DD robustness more precisely. Reports up to roughly 40% and 35% I-RR improvements over existing DD methods under white-box and black-box attacks respectively on CIFAR-10/100.", "abstract": null, "awesome_sections": [ "Applications / Robustness" ], "notes": "Full author list not confirmed beyond the first author (Zheng Zhou)." }, { "id": "robust-dataset-condensation-using-2025", "type": "paper", "title": "Robust Dataset Condensation using Supervised Contrastive Learning", "short": "RDC", "arxiv": null, "url": "https://openaccess.thecvf.com/content/ICCV2025/html/Kim_Robust_Dataset_Condensation_using_Supervised_Contrastive_Learning_ICCV_2025_paper.html", "date": "2025-01-01", "venue": { "name": "ICCV 2025", "status": "conference" }, "authors": [ "Nicole Hee-Yeon Kim" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "applications": [ "noisy-labels" ], "tier": "notable", "links": { "code": "https://github.com/DISL-Lab/RDC-ICCV2025", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "cvf", "depth": "abstract" }, "summary": "RDC is an end-to-end dataset-condensation method that mitigates label noise without a separate noise-reduction preprocessing step, using a \"golden MixUp contrast\" that applies contrastive learning tailored for robust condensation: synthetic samples sharpen class boundaries and reduce noisy representations, while an augmentation strategy pulls clean samples out of the noisy training data to enrich the synthetic images with real-data diversity. Reports outperforming other condensation approaches on CIFAR-10/100 across symmetric, asymmetric and real-world noise.", "abstract": null, "awesome_sections": [ "Applications / Learning with Noisy Labels" ], "notes": "Full author list not confirmed beyond the first author." }, { "id": "towards-universal-dataset-distillation-2025", "type": "paper", "title": "Towards Universal Dataset Distillation via Task-Driven Diffusion", "short": "UniDD", "arxiv": null, "url": "https://openaccess.thecvf.com/content/CVPR2025/html/Qi_Towards_Universal_Dataset_Distillation_via_Task-Driven_Diffusion_CVPR_2025_paper.html", "date": "2025-01", "venue": { "name": "CVPR 2025", "status": "conference" }, "authors": [ "Ding Qi", "Jian Li", "Junyao Gao", "Shuguang Dou", "Ying Tai", "Jianlong Hu", "Bo Zhao", "Yabiao Wang", "Chengjie Wang", "Cairong Zhao" ], "kind": "method", "settings": [ "image-classification", "dense-prediction" ], "families": [ "diffusion-guided-synthesis" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "cvf", "depth": "abstract" }, "summary": "UniDD extends dataset distillation beyond classification to detection and segmentation with a task-driven diffusion model, first mining task-relevant information by training task-specific proxy models (Universal Task Knowledge Mining), then using those proxies to guide a diffusion process that generates task-specific synthetic images with localized, region-level control rather than the global, single-class updates of prior generation methods (Universal Task-Driven Diffusion). The paper reports consistently outperforming state-of-the-art methods on ImageNet-1K, Pascal VOC and MS COCO across classification, detection and segmentation.", "abstract": null, "notes": "Non-arXiv (CVF page blocked for direct fetch); title, authors, mechanism and benchmarks confirmed via independent search excerpts of the paper. Settings expanded from the draft's image-classification-only to include dense-prediction, since the paper's central claim is extending distillation to detection and segmentation." }, { "id": "survey-shang-2025", "type": "paper", "title": "Dataset Distillation in the Era of Large-Scale Data: Methods, Analysis, and Future Directions", "short": "Shang et al. survey", "arxiv": null, "url": "https://www.techrxiv.org/doi/full/10.36227/techrxiv.174785525.52679852/v1", "date": "2025-01", "venue": { "name": "arXiv 2025", "status": "preprint" }, "authors": [ "Xinyi Shang" ], "kind": "survey", "settings": [ "image-classification", "vision-language" ], "families": [], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "abstract" }, "summary": "This survey identifies four shifts since 2023 that earlier surveys predate: scaling distillation to large datasets, moving from optimizing distilled samples alone to improving the whole distillation pipeline, developing theoretical and empirical understanding of core mechanisms, and expanding from images to diverse modalities. It analyzes methods for each stage, benchmarks contemporary methods on large-scale datasets, summarizes distillation across unimodal and multimodal data, and lays out open challenges and directions.", "abstract": null, "notes": "Non-arXiv (TechRxiv preprint, blocked for direct fetch); title, authors and scope confirmed via independent search excerpts of the preprint. Author list in the draft shows only \"Xinyi Shang\"; the full author list was not independently verified." }, { "id": "point-cloud-dd-2025", "type": "paper", "title": "Point Cloud Dataset Distillation", "short": "Point Cloud DD", "arxiv": null, "url": "https://openreview.net/forum?id=Us8v5tDOFd", "date": "2025-01", "venue": { "name": "ICML 2025", "status": "conference" }, "authors": [ "Deyu Bo", "Xinchao Wang" ], "kind": "method", "settings": [ "other-data" ], "families": [ "generator-as-dataset" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "openreview", "depth": "abstract" }, "summary": "This paper adapts dataset distillation to unstructured 3D point clouds by theoretically showing that matching rotation-invariant features between real and synthetic data matters for 3D distillation, introducing a plug-and-play point cloud rotator that aligns point clouds to a canonical orientation, and replacing fixed-size synthetic data with a point-wise generator that produces point clouds at multiple resolutions instead of optimizing one fixed set directly.", "abstract": null, "notes": "Non-arXiv (OpenReview access was blocked for direct fetch); title, authors and mechanism confirmed via independent search excerpts of the ICML listing." }, { "id": "igd-2025", "type": "paper", "title": "Influence-Guided Diffusion for Dataset Distillation", "short": "IGD", "arxiv": null, "url": "https://openreview.net/forum?id=0whx8MhysK", "date": "2025-01", "venue": { "name": "ICLR 2025", "status": "conference" }, "authors": [ "Mingyang Chen", "Jiawei Du", "Bo Huang", "Yi Wang", "Xiaobo Zhang", "Wei Wang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "labels": "soft-relabel", "prior": [ "diffusion" ], "regime": "training-free", "scale": [ "imagenet-1k" ] }, "tier": "notable", "links": { "code": "https://github.com/mchen725/DD_IGD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "full-text" }, "summary": "Frames dataset distillation as controlled diffusion generation guided by a trajectory influence function that links generated samples to downstream training effectiveness, requiring no diffusion retraining; reports 60.3% top-1 at IPC=50 on ImageNet-1K, state-of-the-art among diffusion-based methods.", "abstract": null, "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "explained": { "before": "Pixel-level bilevel methods (DM, IDC-1) don't scale to high-resolution ImageNet; SRe2L/G-VBSM/RDED (decoupled synthesis) and Minimax Diffusion/D4M (diffusion sampling/fine-tuning) address ImageNet scale, but Minimax's fine-tuning approach only reshapes the generator's overall distribution toward representativeness/diversity, without any signal tying a specific generated sample to its actual training value; the paper cites core-set selection literature (Killamsetty et al., Iyer et al.) showing even real data sampled from the true distribution contributes unevenly to training, casting doubt on whether representativeness/diversity fine-tuning alone is \"optimally tailored for effective training.\"", "problem": "No existing diffusion-based distillation method has a way to steer individual sample generation toward samples that are specifically useful for training a downstream classifier, as opposed to merely representative of and diverse within the real distribution — a gap the paper diagnoses by pointing out that diffusion fine-tuning (Minimax) optimizes distributional properties, not a training-effectiveness objective per sample.", "idea": "Reframe dataset distillation as guided diffusion sampling under a \"generalized training-effective condition,\" using the trajectory influence function (TracIn) as the guidance signal: TracIn's formulation happens to reduce to an aggregate over training-loss drops on real data caused by gradient alignment with candidate samples, which the paper shows is (with substitutions) equivalent to the general dataset-distillation objective — so guiding sampling to maximize this influence proxy steers generation toward samples whose training effect approximates that objective, without retraining the diffusion model.", "method": "A pretrained DiT (ImageNet-1K-trained, Stable Diffusion's VAE for the latent encoder/decoder, DDIM sampling with 50 steps) is left frozen; the class-conditional reverse process is modified at each step with two extra guidance gradients (Eq. 9): influence guidance $\\mathcal{G}_I(\\hat{z}_{0|t})$ (Eq. 7), a cosine similarity between the class-averaged gradient of a surrogate ConvNet-6 (trained on the real target dataset for 50 epochs, with representative checkpoints $\\mathcal{R}$ chosen by a gradient- similarity filter rather than fixed intervals) computed on the decoded candidate $D(\\hat z_{0|t})$ versus on real class data, approximating TracIn's trajectory-influence sum without needing to retrain any model on synthetic data at each sampling step; and deviation guidance $\\mathcal{G}_D(z_t)$ (Eq. 8), which pushes each new latent away from the most-similar previously-generated latent for that class (stored in a per-class memory $\\mathcal{M}^c$) to prevent redundancy. Guidance is applied only during an early-to-mid window of denoising steps (e.g. $t\\in[30,45]$ of 50 DDIM steps) — \"early-stage guidance\" — because full-range guidance with a large scale $k$ overfits to the surrogate and produces visible image artifacts. The framework composes with either a raw pretrained DiT (\"DiT-IGD\") or a Minimax-fine-tuned DiT (\"Minimax-IGD\"), since it only modifies sampling, not training.", "evidence": "ImageNette/ImageWoof, hard labels, cross-architecture (ConvNet-6/ResNetAP-10/ResNet-18), Table 1: at IPC>=50, DiT-IGD improves over raw DiT by 5.8% (Nette) and 6.6% (Woof) on average; Minimax-IGD improves over Minimax by 4.7% (Nette) and 5.1% (Woof); e.g. Woof/ResNetAP-10/IPC=50: Minimax-IGD 65.0% vs. Minimax 59.8% vs. DiT-IGD 62.7% vs. raw DiT 55.1%. ImageNet-1K (Table 2, RDED's protocol, soft-label teacher relabeling with a real-data-trained ResNet-18): IPC=50 Minimax-IGD 60.3% vs. RDED 56.5% vs. G-VBSM 51.8% vs. SRe2L 46.8% vs. raw DiT 52.9% (a +4.0-point gain over RDED, the state-of-the-art image-based baseline, cited in the abstract); IPC=10 DiT-IGD 45.5% vs. DiT 39.6% (+5.9). Cross- architecture on ImageNet-1K (Table 3, unseen ResNet101/MobileNet-V2/EfficientNet-B0/Swin-T): DiT-IGD and Minimax-IGD beat RDED by an average 4.6% and 5.0% at IPC=50. Key ablations: (1) Table 5 — on ImageNette/ResNet-18, adding only influence guidance to raw DiT gives 76.5%/79.1% (IPC50/100) vs. 75.2%/77.8% baseline; only deviation guidance gives 78.2%/80.7%; combining both gives the best 81.0%/84.4% — the two guidance terms are complementary, and deviation guidance alone recovers much of Minimax's diversity benefit without any fine-tuning; for the already-fine-tuned Minimax backbone, influence guidance alone nearly closes the gap to the full method (81.5% vs. 82.0% at IPC50), showing influence guidance matters most when diversity is already handled; (2) Fig. 2b/2c — applying guidance across the entire denoising range with large $k$ reduces the influence loss further than early-stage guidance but *drops* validation accuracy and produces visible artifacts (Fig. 2a), confirming that over-guiding overfits to the ConvNet-6 surrogate rather than improving general training utility; (3) Table 6 — the proposed gradient-similarity checkpoint selection beats regular-interval selection at matched checkpoint counts (e.g. 4 similarity-selected checkpoints beat 10 regularly-spaced ones, 82.0% vs. 81.1%), and the selected indices cluster in early training epochs.", "limitations": "The DiT backbone is pretrained on ImageNet-1K, i.e. matched to the target distribution for every reported experiment (ImageNette, ImageWoof, ImageNet-1K are all ImageNet-derived); the influence- guidance mechanism itself additionally requires training a surrogate ConvNet-6 on the real target dataset for 50 epochs purely to compute guidance gradients, which is a real-data-dependent cost the paper does not fold into its \"no diffusion retraining\" framing; the ImageNet-1K headline numbers use teacher soft-label relabeling (RDED's protocol) while the ImageNette/Woof numbers use hard labels, so the two tables are not on a common label regime; the paper has no explicit stated-limitations section, and does not test IPC=1 or CIFAR/MNIST-scale settings, nor a diffusion model pretrained on non-target data." } }, { "id": "condenselm-2025", "type": "paper", "title": "CondenseLM: LLMs-driven Text Dataset Condensation via Reward Matching", "short": "CondenseLM", "arxiv": null, "url": "https://aclanthology.org/2025.emnlp-main.65/", "date": "2025-01", "venue": { "name": "EMNLP 2025", "status": "conference" }, "authors": [ "Cheng Shen", "Yew-Soon Ong", "Joey Tianyi Zhou" ], "kind": "method", "settings": [ "text" ], "families": [], "tier": "catalogued", "links": { "code": "https://github.com/cs6331/CondenseLM/", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "aclanthology", "depth": "abstract" }, "summary": "Proposes CondenseLM, which uses an LLM-driven pipeline guided by reward matching to generate condensed text-classification datasets directly rather than optimizing embedding-level synthetic samples, aiming to improve representability and coverage over prior text condensation methods; reports it substantially outperforms coreset-selection and dataset-condensation baselines on SST-2, MNLI, AG News and IMDB while reducing computational cost.", "abstract": null, "awesome_sections": [ "Applications / Text" ], "notes": "Mechanism does not map onto the P1-P4 matching families; left empty." }, { "id": "video-dc-study-2024", "type": "paper", "title": "A Large-Scale Study on Video Action Dataset Condensation", "short": "Video DC study", "arxiv": "2412.21197", "url": null, "date": "2024-12-30", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Yang Chen", "Sheng Guo", "Bo Zheng", "Limin Wang" ], "kind": "evaluation-benchmark", "settings": [ "video" ], "families": [ "coreset-selection", "decoupled-synthesis", "trajectory-matching" ], "builds_on": [ "video-sdd-2024", "rded-2024", "edc-2024", "datm-2024" ], "tier": "catalogued", "links": { "code": "https://github.com/MCG-NJU/Video-DC", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "This large-scale study systematically examines video dataset condensation along three axes: temporal processing of video data, the evaluation protocol, and how condensation algorithms adapt to the space-time domain. It finds that labeling method matters a great deal for condensation performance, that simple sliding-window sampling suffices for temporal processing, and that dataset-distillation methods win in harder scenarios while sample-selection methods win in easier ones; it also proposes a unified evaluation protocol and reports state-of-the-art results on HMDB51, UCF101, SSv2 and K400 under it.", "abstract": "Recently, dataset condensation has made significant progress in the image domain. Unlike images, videos possess an additional temporal dimension, which harbors considerable redundant information, making condensation even more crucial. However, video dataset condensation still remains an underexplored area. We aim to bridge this gap by providing a large-scale study with systematic design and fair comparison. Specifically, our work delves into three key aspects to provide valuable empirical insights: (1) temporal processing of video data, (2) the evaluation protocol for video dataset condensation, and (3) adaptation of condensation algorithms to the space-time domain. From this study, we derive several intriguing observations: (i) labeling methods greatly influence condensation performance, (ii) simple sliding-window sampling is effective for temporal processing, and (iii) dataset distillation methods perform better in challenging scenarios, while sample selection methods excel in easier ones. Furthermore, we propose a unified evaluation protocol for the fair comparison of different condensation algorithms and achieve state-of-the-art results on four widely-used action recognition datasets: HMDB51, UCF101, SSv2 and K400. Our code is available at https://github.com/MCG-NJU/Video-DC.", "notes": "", "explained": { "before": "Wang et al. (video-sdd-2024) gave the first systematic study of video distillation but used only small-scale HMDB51/MiniUCF, segment-based temporal sampling, and did not investigate how evaluation settings (labeling, augmentation, loss) affect reported condensation performance — mirroring the label-regime confound already documented for image condensation (RDED, EDC, DATM, SRe2L-family methods), which this paper imports into the video setting essentially unstudied.", "problem": "Because video condensation papers each pick their own temporal processing, labeling scheme and training recipe, it is unclear whether reported gaps between methods reflect the condensation algorithm itself or the evaluation protocol around it, and no work had systematically adapted state-of-the-art image condensation methods (distribution/statistical matching, trajectory matching, realism-based selection) to video or tested them at the scale of Kinetics-400/SSv2.", "idea": "Treat video condensation as three separable design axes — temporal processing (sampling + interpolation), the condensation algorithm, and the evaluation protocol (labels, augmentation, loss) — and vary each independently to isolate which one actually drives reported accuracy differences, proposing sliding-window sampling and a unified evaluation protocol as the fixes.", "method": "Adapts three image condensation methods to video by treating clips as an extra time axis: RDED (training-free realism/diversity-scored real-patch selection, extended by concatenating video tubes spatially), EDC (SRe2L-style statistical/BN-moment matching, extended to space-time statistics), and DATM (multi-step trajectory matching with a difficulty-increasing curriculum). Introduces sliding-window sampling (overlapping clips sampled sequentially along time, replacing segment sampling) and compares interpolation schemes (none, duplication, linear, learnable) for aligning condensed clip length to the evaluation model's input length. Systematically ablates labeling regime (hard, single soft label, Multi-SL — multiple augmented views of one clip each with their own teacher logits, as in SRe2L/G-VBSM/EDC), CutMix augmentation, and loss function (KL vs. MSE against ground-truth), then fixes a single evaluation protocol (Multi-SL, no CutMix, MSE-GT loss) for its head-to-head comparisons.", "evidence": "Table 8 (HMDB51/UCF101 top-1, SSv2/K400 top-5, soft labels, MiniC3D eval, various IPC): the paper's reimplementations outperform video-sdd-2024's numbers at matched IPC (e.g. HMDB51 IPC=1, DATM 9.0% vs. video-sdd-2024's best prior 8.6%; UCF101 IPC=5, DATM 20.8% vs. video-sdd-2024 not reported at that scale) and reach new state-of-the-art on all four datasets, with RDED best on HMDB51/UCF101 and DATM best on the larger, more class-diverse SSv2/K400 (full-dataset upper bounds: HMDB51 26.1%, UCF101 52.6%, SSv2 47.5%, K400 44.6%). Figure 4 / Tables 2-3 (UCF101, IPC=1): labeling method changes accuracy by up to 8.6 points — more than augmentation (3.3 pts) or loss function (2.4 pts) combined — and can reverse method rankings (DATM without CutMix, 12.1%, beats RDED with CutMix, 11.0%, under soft labels). Table 4 (K400): sliding-window sampling beats segment sampling for all three methods (e.g. RDED 11.5% vs. 10.6%); Table 5 (SSv2): all training-free interpolators (duplication, linear) hurt accuracy relative to no interpolation. Cost: DATM uses up to 40GB GPU memory vs. EDC's under 10GB (Sec. 4.4), with no wall-clock numbers given.", "limitations": "Does not propose or test a trainable interpolator despite showing training-free ones fail; explicitly excludes generative/diffusion-based video condensation methods \"due to limitation of time and energy\"; evaluation still relies on relatively small backbones (MiniC3D primary, R(2+1)D/I3D/SlowOnly for transfer) rather than modern video transformers; DATM's high memory cost (up to 40GB) is flagged as limiting its scalability but not resolved." } }, { "id": "fedvck-2025", "type": "paper", "title": "FedVCK: Non-IID Robust and Communication-Efficient Federated Learning via Valuable Condensed Knowledge for Medical Image Analysis", "short": "FedVCK", "arxiv": "2412.18557", "url": null, "date": "2024-12-24", "venue": { "name": "AAAI 2025", "status": "conference" }, "authors": [ "Guochen Yan", "Luyuan Xie", "Xinyi Gao", "Wentao Zhang", "Qingni Shen", "Yuejian Fang", "Zhonghai Wu" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "applications": [ "federated-learning", "medical" ], "tier": "notable", "links": { "code": "https://github.com/Youth-49/FedVCK_2024", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "FedVCK condenses each client's data with latent distribution constraints to improve knowledge quality, and each round selectively condenses only knowledge the current global model has not yet assimilated, cutting redundant communication; on the server, relational supervised contrastive learning gives extra supervision for the global update. Reports outperforming state-of-the-art federated learning methods on medical tasks under non-IID data and limited communication budgets.", "abstract": "Federated learning has become a promising solution for collaboration among medical institutions. However, data owned by each institution would be highly heterogeneous and the distribution is always non-independent and identical distribution (non-IID), resulting in client drift and unsatisfactory performance. Despite existing federated learning methods attempting to solve the non-IID problems, they still show marginal advantages but rely on frequent communication which would incur high costs and privacy concerns. In this paper, we propose a novel federated learning method: \\textbf{Fed}erated learning via \\textbf{V}aluable \\textbf{C}ondensed \\textbf{K}nowledge (FedVCK). We enhance the quality of condensed knowledge and select the most necessary knowledge guided by models, to tackle the non-IID problem within limited communication budgets effectively. Specifically, on the client side, we condense the knowledge of each client into a small dataset and further enhance the condensation procedure with latent distribution constraints, facilitating the effective capture of high-quality knowledge. During each round, we specifically target and condense knowledge that has not been assimilated by the current model, thereby preventing unnecessary repetition of homogeneous knowledge and minimizing the frequency of communications required. On the server side, we propose relational supervised contrastive learning to provide more supervision signals to aid the global model updating. Comprehensive experiments across various medical tasks show that FedVCK can outperform state-of-the-art methods, demonstrating that it's non-IID robust and communication-efficient.", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "" }, { "id": "adaptive-dataset-quantization-2025", "type": "paper", "title": "Adaptive Dataset Quantization", "arxiv": "2412.16895", "url": null, "date": "2024-12-22", "venue": { "name": "AAAI 2025", "status": "conference" }, "authors": [ "Muquan Li", "Dongyang Zhang", "Qiang Dong", "Xiurui Xie", "Ke Qin" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "dataset-quantization" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Replaces Dataset Quantization's uniform bin sampling with an adaptive strategy that scores each generated bin's representativeness (texture level), diversity (contrastive-learning based) and importance, then samples accordingly; reports an average 3% accuracy gain over DQ across benchmarks with better cross-architecture generalization.", "abstract": "Contemporary deep learning, characterized by the training of cumbersome neural networks on massive datasets, confronts substantial computational hurdles. To alleviate heavy data storage burdens on limited hardware resources, numerous dataset compression methods such as dataset distillation (DD) and coreset selection have emerged to obtain a compact but informative dataset through synthesis or selection for efficient training. However, DD involves an expensive optimization procedure and exhibits limited generalization across unseen architectures, while coreset selection is limited by its low data keep ratio and reliance on heuristics, hindering its practicality and feasibility. To address these limitations, we introduce a newly versatile framework for dataset compression, namely Adaptive Dataset Quantization (ADQ). Specifically, we first identify the sub-optimal performance of naive Dataset Quantization (DQ), which relies on uniform sampling and overlooks the varying importance of each generated bin. Subsequently, we propose a novel adaptive sampling strategy through the evaluation of generated bins' representativeness score, diversity score and importance score, where the former two scores are quantified by the texture level and contrastive learning-based techniques, respectively. Extensive experiments demonstrate that our method not only exhibits superior generalization capability across different architectures, but also attains state-of-the-art results, surpassing DQ by average 3\\% on various datasets.", "builds_on": [ "dq-2023" ], "awesome_sections": [ "Main / Dataset Quantization" ] }, { "id": "cmi-2025", "type": "paper", "title": "Going Beyond Feature Similarity: Effective Dataset Distillation based on Class-Aware Conditional Mutual Information", "short": "CMI", "arxiv": "2412.09945", "url": null, "date": "2024-12-13", "venue": { "name": "ICLR 2025", "status": "conference" }, "authors": [ "Xinhao Zhong", "Bin Chen", "Hao Fang", "Xulin Gu", "Shu-Tao Xia", "En-Hui Yang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/ndhg1213/CMIDD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Introduces conditional mutual information, estimated in the feature space of pretrained networks, as a class-aware complexity regularizer added to an existing dataset-distillation loss to counter synthetic sets that are needlessly hard to learn from, reporting improved accuracy and training efficiency as a general add-on to distribution-matching-based methods.", "abstract": "Dataset distillation (DD) aims to minimize the time and memory consumption needed for training deep neural networks on large datasets, by creating a smaller synthetic dataset that has similar performance to that of the full real dataset. However, current dataset distillation methods often result in synthetic datasets that are excessively difficult for networks to learn from, due to the compression of a substantial amount of information from the original data through metrics measuring feature similarity, e,g., distribution matching (DM). In this work, we introduce conditional mutual information (CMI) to assess the class-aware complexity of a dataset and propose a novel method by minimizing CMI. Specifically, we minimize the distillation loss while constraining the class-aware complexity of the synthetic dataset by minimizing its empirical CMI from the feature space of pre-trained networks, simultaneously. Conducting on a thorough set of experiments, we show that our method can serve as a general regularization method to existing DD methods and improve the performance and training efficiency.", "awesome_sections": [ "Main / Better Optimization" ], "builds_on": [ "dm-2023" ] }, { "id": "ddm-2025", "type": "paper", "title": "Decomposed Distribution Matching in Dataset Condensation", "short": "DDM", "arxiv": "2412.04748", "url": null, "date": "2024-12-06", "venue": { "name": "WACV 2025", "status": "conference" }, "authors": [ "Sahar Rahimi Malakshan", "Mohammad Saeed Ebrahimi Saadabadi", "Ali Dabouei", "Nasser M. Nasrabadi" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "applications": [ "continual-learning" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "parameterization": [ "pixels" ], "labels": "hard", "prior": [ "random-networks" ], "regime": "single-level", "scale": [ "mnist-cifar", "tiny-imagenet", "imagenet-1k", "imagenet-subsets" ] }, "explained": { "before": "dm-2023 matches only class-wise mean features in random networks; cafe-2022 aligns richer multi-scale features but reverts to a bi-level scheme; idm-2023 and datadam-2023 improve the embeddings or add spatial attention supervision but, the paper argues, still leave style un-modeled and diversity unconstrained.", "problem": "Diagnoses dm-2023's accuracy gap to bi-level methods (an 8-point drop versus dsa-2021 on CIFAR-100 at IPC=10) by decomposing the feature distribution into content and style: (1) a style discrepancy -- t-SNE of first-layer feature-map moments shows condensed and real images of the same class have visibly different style statistics, and drifting Herding's real-image style toward DM's measurably hurts downstream accuracy -- and (2) limited intra-class diversity, since DM's synthetic samples cluster locally in the final-layer embedding rather than spanning the class.", "idea": "Add two loss terms on top of DM's mean-matching objective, in the same random-network, single-level framework: a style-matching term using well-established style indicators (channel-wise feature-map moments and Gram-matrix correlations, as in neural style transfer) to close the style gap, and an intra-class diversity term that pushes each synthetic sample's embedding away from its k-nearest synthetic neighbors of the same class via a KL-divergence penalty.", "method": "For randomly sampled networks (as in dm-2023), computes a Moments-Matching loss (MSE between per-layer channel-wise mean and variance of real vs. synthetic feature maps, following AdaIN) and a Correlation- Matching loss (MSE between per-layer Gram matrices, following Gatys-style texture/style representations), summed as the style loss $L_S = \\alpha L_{MM} + L_{CM}$. An Intra-Class Diversity (ICD) loss maximizes the KL divergence between each synthetic sample's softmax feature and the mean feature of its k=0.2*IPC nearest synthetic same-class neighbors. Total objective $\\lambda L_S + (\\beta L_{ICD} + L_{MMD})$ is optimized by gradient descent on the synthetic pixel images only; no network is trained during synthesis. Hard labels; differentiable Siamese augmentation as in DM.", "evidence": "CIFAR-10, IPC=10, ConvNet: DDM 53.0% vs. DM 48.9% (+4.1pp, Table 1); IPC=50: 65.6% vs. 63.0% (+2.6pp). CIFAR-100, IPC=10: 33.9% vs. 29.7% (+4.2pp). Tiny-ImageNet, IPC=50: 27.4% vs. 25.3% (+2.1pp). ImageNet-1K (64x64), IPC=50: 15.6% vs. 11.4% (+4.2pp). ImageWoof/ImageNette (128x128, IPC=10): 34.5%/58.1% vs. 31.2%/55.6% (Table 3). ResNet-18 (Table 2, CIFAR-10, IPC=10): 40.9% vs. DM 32.6% (+8.3pp) -- a larger gain than on ConvNet, suggesting the fix matters more for architectures more sensitive to style. Orthogonality (Table 5): adding the style-matching module alone to DataDAM and IDM improves both (e.g. IDM CIFAR-100 IPC=10: 45.1%->47.2%). Cross-architecture (CIFAR-10, IPC=10, trained on ConvNet): DDM 53.0/48.7/46.2/42.6% on ConvNet/AlexNet/VGG-11/ResNet-18 vs. DM 48.6/38.3/40.8/39.2% (Table 4). Continual learning (CIFAR-100, 5-step, IPC-based memory): 39.9% final accuracy vs. DM 34.4%, DSA 31.7%. Ablation (Fig. 3a, CIFAR-10, IPC=10): DM+MM and DM+CM each improve over DM alone, DM+MM+CM is better than either alone, and adding ICD gives a further gain, with the paper noting ICD is irrelevant at IPC=1 (no intra-class diversity to exploit) where the accuracy gain (Table 1) is attributed entirely to style matching.", "limitations": "No GPU-hour, wall-clock or memory comparison is reported anywhere in the paper despite adding two extra loss terms (per-layer moments, per-layer Gram matrices over multiple layers, plus a k-NN search for ICD) on top of DM's single MMD term, so the added compute cost relative to plain DM is not quantified. Hard labels only; no soft-label or teacher-relabeling variant is tested, unlike most other 2024-2025 entries in this group. Evaluated only against DM-family baselines (DM, CAFE, DataDAM, IDM) plus DD/DG/DSA and coreset selection; no comparison to trajectory matching (MTT) or to the richer-statistics distribution-matching methods (M3D, WMDD, NCFM) that were concurrent or slightly earlier. Random networks only (no pretrained or partially trained embeddings), so it does not test whether style matching helps or is redundant once embeddings already carry more structure, as in DANCE or WMDD." }, "summary": "Decomposes the dataset distribution that distribution matching targets into content and style, and identifies two shortcomings of the plain distribution-matching objective: a style mismatch between real and condensed data, and limited intra-class diversity in the condensed set. Matches per-layer feature-map statistical moments as a style term and maximizes intra-class KL divergence among synthetic samples as a content-diversity term, reporting accuracy gains of up to 4.1% on CIFAR-10, 4.2% on CIFAR-100, 4.3% on Tiny-ImageNet, 2.0% on ImageNet-1K, 3.3% on ImageWoof, 2.5% on ImageNette, and 5.5% in continual-learning accuracy over the distribution-matching baseline.", "abstract": "Dataset Condensation (DC) aims to reduce deep neural networks training efforts by synthesizing a small dataset such that it will be as effective as the original large dataset. Conventionally, DC relies on a costly bi-level optimization which prohibits its practicality. Recent research formulates DC as a distribution matching problem which circumvents the costly bi-level optimization. However, this efficiency sacrifices the DC performance. To investigate this performance degradation, we decomposed the dataset distribution into content and style. Our observations indicate two major shortcomings of: 1) style discrepancy between original and condensed data, and 2) limited intra-class diversity of condensed dataset. We present a simple yet effective method to match the style information between original and condensed data, employing statistical moments of feature maps as well-established style indicators. Moreover, we enhance the intra-class diversity by maximizing the Kullback-Leibler divergence within each synthetic class, i.e., content. We demonstrate the efficacy of our method through experiments on diverse datasets of varying size and resolution, achieving improvements of up to 4.1% on CIFAR10, 4.2% on CIFAR100, 4.3% on TinyImageNet, 2.0% on ImageNet-1K, 3.3% on ImageWoof, 2.5% on ImageNette, and 5.5% in continual learning accuracy.", "notes": "", "builds_on": [ "dm-2023", "cafe-2022", "idm-2023", "datadam-2023" ] }, { "id": "krr-provable-2024", "type": "paper", "title": "Provable and Efficient Dataset Distillation for Kernel Ridge Regression", "short": "Provable KRR DD", "arxiv": null, "url": "https://openreview.net/forum?id=WI2VpcBdnd", "date": "2024-12", "venue": { "name": "NeurIPS 2024", "status": "conference" }, "authors": [ "Yilan Chen", "Wei Huang", "Tsui-Wei Weng" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "kernel-closed-form" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "openreview", "depth": "abstract" }, "summary": "Proves that for kernel ridge regression one data point per class is necessary and sufficient to exactly recover the original model in many settings, derives necessary and sufficient conditions for exact recovery with linear and surjective-feature-map kernels, and shows k+1 points suffice for deep linear networks with k classes; the resulting closed-form construction outperforms KIP while reported up to 15,840x faster on CIFAR-100.", "abstract": null, "awesome_sections": [ "Main / Kernel-Based Distillation" ], "builds_on": [ "kip-2021" ] }, { "id": "fairdd-2025", "type": "paper", "title": "FairDD: Fair Dataset Distillation", "short": "FairDD", "arxiv": "2411.19623", "url": null, "date": "2024-11-29", "venue": { "name": "NeurIPS 2025", "status": "conference" }, "authors": [ "Qihang Zhou", "Shenhao Fang", "Shibo He", "Wenchao Meng", "Jiming Chen" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "applications": [ "fairness" ], "tier": "notable", "links": { "code": "https://github.com/zqhang/FairDD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Shows that dataset distillation does not fix, and typically worsens, unfairness toward protected-attribute minority groups because matching-based objectives align synthetic data to the whole distribution, which majority groups dominate; FairDD instead synchronously matches synthetic data to each protected-attribute group separately, without changing the underlying matching architecture. Reported to significantly improve fairness with a favorable accuracy trade-off across both distribution-matching and gradient-matching DD methods.", "abstract": "Condensing large datasets into smaller synthetic counterparts has demonstrated its promise for image classification. However, previous research has overlooked a crucial concern in image recognition: ensuring that models trained on condensed datasets are unbiased towards protected attributes (PA), such as gender and race. Our investigation reveals that dataset distillation fails to alleviate the unfairness towards minority groups within original datasets. Moreover, this bias typically worsens in the condensed datasets due to their smaller size. To bridge the research gap, we propose a novel fair dataset distillation (FDD) framework, namely FairDD, which can be seamlessly applied to diverse matching-based DD approaches (DDs), requiring no modifications to their original architectures. The key innovation of FairDD lies in synchronously matching synthetic datasets to PA-wise groups of original datasets, rather than indiscriminate alignment to the whole distributions in vanilla DDs, dominated by majority groups. This synchronized matching allows synthetic datasets to avoid collapsing into majority groups and bootstrap their balanced generation to all PA groups. Consequently, FairDD could effectively regularize vanilla DDs to favor biased generation toward minority groups while maintaining the accuracy of target attributes. Theoretical analyses and extensive experimental evaluations demonstrate that FairDD significantly improves fairness compared to vanilla DDs, with a promising trade-off between fairness and accuracy. Its consistent superiority across diverse DDs, spanning Distribution and Gradient Matching, establishes it as a versatile FDD approach. Code is available at https://github.com/zqhang/FairDD.", "awesome_sections": [ "Applications / Fairness" ], "notes": "" }, { "id": "delt-2025", "type": "paper", "title": "DELT: A Simple Diversity-driven EarlyLate Training for Dataset Distillation", "short": "DELT", "arxiv": "2411.19946", "url": null, "date": "2024-11-29", "venue": { "name": "CVPR 2025", "status": "conference" }, "authors": [ "Zhiqiang Shen", "Ammar Sherif", "Zeyuan Yin", "Shitong Shao" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "tier": "notable", "links": { "code": "https://github.com/VILA-Lab/DELT", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Diagnoses a diversity problem in batch-to-global (decoupled) matching methods such as SRe2L, G-VBSM, WMDD and CDA, where images optimized independently under the same reused global supervision end up homogeneous; DELT partitions the images-per-class budget into subtasks distilled at distinct early and late optimization phases, then generalizes back to the full task, reporting 2-5% average accuracy gains, 5%+ higher per-class diversity, and up to 39.3% less synthesis time on CIFAR, Tiny-ImageNet, ImageNet-1K and its sub-datasets.", "abstract": "Recent advances in dataset distillation have led to solutions in two main directions. The conventional batch-to-batch matching mechanism is ideal for small-scale datasets and includes bi-level optimization methods on models and syntheses, such as FRePo, RCIG, and RaT-BPTT, as well as other methods like distribution matching, gradient matching, and weight trajectory matching. Conversely, batch-to-global matching typifies decoupled methods, which are particularly advantageous for large-scale datasets. This approach has garnered substantial interest within the community, as seen in SRe$^2$L, G-VBSM, WMDD, and CDA. A primary challenge with the second approach is the lack of diversity among syntheses within each class since samples are optimized independently and the same global supervision signals are reused across different synthetic images. In this study, we propose a new Diversity-driven EarlyLate Training (DELT) scheme to enhance the diversity of images in batch-to-global matching with less computation. Our approach is conceptually simple yet effective, it partitions predefined IPC samples into smaller subtasks and employs local optimizations to distill each subset into distributions from distinct phases, reducing the uniformity induced by the unified optimization process. These distilled images from the subtasks demonstrate effective generalization when applied to the entire task. We conduct extensive experiments on CIFAR, Tiny-ImageNet, ImageNet-1K, and its sub-datasets. Our approach outperforms the previous state-of-the-art by 2$\\sim$5% on average across different datasets and IPCs (images per class), increasing diversity per class by more than 5% while reducing synthesis time by up to 39.3% for enhancing the training efficiency. Code is available at: https://github.com/VILA-Lab/DELT.", "builds_on": [ "sre2l-2023", "g-vbsm-2024", "wmdd-2025", "cda-2024" ], "awesome_sections": [ "Main / Decoupled Distillation" ], "explained": { "before": "Decoupled methods (SRe2L, CDA, G-VBSM, WMDD) recover every synthetic image independently against the same global teacher statistics (batch-norm running means/variances and class logits). Bi-level matching methods (FRePo, RCIG, RaT-BPTT, gradient/distribution/trajectory matching) supervise with local real batches but do not scale to ImageNet-1K. G-VBSM increased diversity by adding backbones and statistics, at a large cost in synthesis time.", "problem": "Batch-to-global matching reuses one supervision signal for all images of a class, so independently optimized images converge to similar solutions; the paper measures this as high intra-class cosine similarity of ResNet-18 features on ImageNet-1K.", "idea": "Split each class's budget into sequential sub-batches that join one shared recovery loop at staggered times. Early-joining batches are optimized for many iterations and drift far from their real-image initialization; late-joining batches stay close to it. The concatenated set therefore spans a range of optimization depths, which adds diversity and saves compute without extra models.", "method": "SRe2L-style recovery (teacher cross-entropy plus batch-norm statistic matching) with a pretrained ResNet-18 teacher, initialized from real images. Two hyperparameters: a maximum iteration count for the first sub-batch (e.g. 4,000) and a round interval after which the next sub-batch joins (e.g. 500), all under one Adam schedule. Students are trained with SRe2L/FKD-style per-crop soft labels from the teacher: AdamW, 300 epochs, RandAugment, RandomResizedCrop and flips.", "evidence": "ImageNet-1K with ResNet-18 students and teacher soft labels (Table 1): 46.1% at IPC 10 and 59.2% at IPC 50, against RDED's 42.0% and 56.5%; 66.1% at IPC 50 with ResNet-101. Recovery time at IPC 50 falls from 29.0 h (SRe2L/CDA) to 17.6 h, a 39.3% reduction (Table 6). Adding the staggered schedule to real-image initialization raises IPC-10 accuracy from 43.5% to 46.1% on ImageNet-1K and from 39.4% to 43.0% on CIFAR-10. No hard-label results are reported. Note: Table 1 quotes SRe2L at 21.3% (IPC 10, ResNet-18), while the paper's own cross-architecture table re-runs SRe2L under CDA's configuration at 41.9%. That gap shows how strongly the evaluation recipe moves decoupled-method numbers.", "limitations": "The authors note a remaining gap to full-data training and possible privacy leakage from images kept close to real initializations. In the experiments the staggered schedule helps little or hurts at IPC 1, where RDED is stronger (e.g. ImageNette, 24.1% vs 35.8%), and RDED is clearly better on ImageNet-100 at IPC 10 across backbones. All results depend on teacher soft labels, and the paper does not report label storage." } }, { "id": "video-set-distillation-information-2024", "type": "paper", "title": "Video Set Distillation: Information Diversification and Temporal Densification", "short": "IDTD", "arxiv": "2412.00111", "url": null, "date": "2024-11-28", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Yinjie Zhao", "Heng Zhao", "Bihan Wen", "Yew-Soon Ong", "Joey Tianyi Zhou" ], "kind": "method", "settings": [ "video" ], "families": [ "dataset-quantization", "parameterization" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "This paper introduces video set distillation, jointly reducing within-sample redundancy (across frames of one video) and inter-sample redundancy (across videos in a set), which prior key-frame selection, dataset pruning or single-video distillation methods each address only one of. Its IDTD method uses a Feature Pool and Feature Selectors to preserve diversity across samples and a Temporal Fusor to keep temporal information dense within each synthesized video, reporting state-of-the-art results in video dataset distillation.", "abstract": "The rapid development of AI models has led to a growing emphasis on enhancing their capabilities for complex input data such as videos. While large-scale video datasets have been introduced to support this growth, the unique challenges of reducing redundancies in video \\textbf{sets} have not been explored. Compared to image datasets or individual videos, video \\textbf{sets} have a two-layer nested structure, where the outer layer is the collection of individual videos, and the inner layer contains the correlations among frame-level data points to provide temporal information. Video \\textbf{sets} have two dimensions of redundancies: within-sample and inter-sample redundancies. Existing methods like key frame selection, dataset pruning or dataset distillation are not addressing the unique challenge of video sets since they aimed at reducing redundancies in only one of the dimensions. In this work, we are the first to study Video Set Distillation, which synthesizes optimized video data by jointly addressing within-sample and inter-sample redundancies. Our Information Diversification and Temporal Densification (IDTD) method jointly reduces redundancies across both dimensions. This is achieved through a Feature Pool and Feature Selectors mechanism to preserve inter-sample diversity, alongside a Temporal Fusor that maintains temporal information density within synthesized videos. Our method achieves state-of-the-art results in Video Dataset Distillation, paving the way for more effective redundancy reduction and efficient AI model training on video datasets.", "notes": "Families are a best-effort fit (selection/diversity mechanism for dataset-quantization, temporal restructuring for parameterization); no P1-P4 matching objective is explicitly named in the abstract." }, { "id": "dataset-distillers-are-good-2024", "type": "paper", "title": "Dataset Distillers Are Good Label Denoisers In the Wild", "short": "DD_LNL", "arxiv": "2411.11924", "url": null, "date": "2024-11-18", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Lechao Cheng", "Kaifeng Chen", "Jiyang Li", "Shengeng Tang", "Shufei Zhang", "Meng Wang" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [], "applications": [ "noisy-labels", "privacy" ], "tier": "catalogued", "links": { "code": "https://github.com/Kciiiman/DD_LNL", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Evaluates three existing dataset distillation methods (DATM, DANCE, RCIG) as an offline, feedback-loop-free alternative to noise evaluation for learning with noisy labels. Reports that distillation denoises effectively under symmetric random noise but struggles with structured asymmetric noise, which can be absorbed into the distilled samples, and that clean tail-class samples can be lossily compressed away.", "abstract": "Learning from noisy data has become essential for adapting deep learning models to real-world applications. Traditional methods often involve first evaluating the noise and then applying strategies such as discarding noisy samples, re-weighting, or re-labeling. However, these methods can fall into a vicious cycle when the initial noise evaluation is inaccurate, leading to suboptimal performance. To address this, we propose a novel approach that leverages dataset distillation for noise removal. This method avoids the feedback loop common in existing techniques and enhances training efficiency, while also providing strong privacy protection through offline processing. We rigorously evaluate three representative dataset distillation methods (DATM, DANCE, and RCIG) under various noise conditions, including symmetric noise, asymmetric noise, and real-world natural noise. Our empirical findings reveal that dataset distillation effectively serves as a denoising tool in random noise scenarios but may struggle with structured asymmetric noise patterns, which can be absorbed into the distilled samples. Additionally, clean but challenging samples, such as those from tail classes in imbalanced datasets, may undergo lossy compression during distillation. Despite these challenges, our results highlight that dataset distillation holds significant promise for robust model training, especially in high-privacy environments where noise is prevalent. The source code is available at https://github.com/Kciiiman/DD_LNL.", "awesome_sections": [ "Applications / Learning with Noisy Labels" ], "notes": "families left empty: the paper evaluates existing methods (DATM=trajectory-matching, RCIG=kernel-closed-form) as a denoising tool rather than proposing a new mechanism." }, { "id": "autopalette-2024", "type": "paper", "title": "Color-Oriented Redundancy Reduction in Dataset Distillation", "short": "AutoPalette", "arxiv": "2411.11329", "url": null, "date": "2024-11-18", "venue": { "name": "NeurIPS 2024", "status": "conference" }, "authors": [ "Bowen Yuan", "Zijian Wang", "Mahsa Baktashmotlagh", "Yadan Luo", "Zi Huang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "color-quantized" ] }, "tier": "catalogued", "links": { "code": "https://github.com/KeViNYuAn0314/AutoPalette", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Reduces color redundancy in distilled images with a palette network that dynamically allocates colors from a reduced palette to each pixel based on the region's importance for training, plus a color-guided initialization strategy that selects real images with the least redundant color patterns across the dataset.", "abstract": "Dataset Distillation (DD) is designed to generate condensed representations of extensive image datasets, enhancing training efficiency. Despite recent advances, there remains considerable potential for improvement, particularly in addressing the notable redundancy within the color space of distilled images. In this paper, we propose AutoPalette, a framework that minimizes color redundancy at the individual image and overall dataset levels, respectively. At the image level, we employ a palette network, a specialized neural network, to dynamically allocate colors from a reduced color space to each pixel. The palette network identifies essential areas in synthetic images for model training and consequently assigns more unique colors to them. At the dataset level, we develop a color-guided initialization strategy to minimize redundancy among images. Representative images with the least replicated color patterns are selected based on the information gain. A comprehensive performance study involving various datasets and evaluation scenarios is conducted, demonstrating the superior performance of our proposed color-aware DD compared to existing DD methods. The code is available at \\url{https://github.com/KeViNYuAn0314/AutoPalette}.", "awesome_sections": [ "Main / Distilled Dataset Parametrization" ] }, { "id": "beard-2024", "type": "paper", "title": "BEARD: Benchmarking the Adversarial Robustness for Dataset Distillation", "short": "BEARD", "arxiv": "2411.09265", "url": null, "date": "2024-11-14", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Zheng Zhou", "Wenquan Feng", "Shuchang Lyu", "Guangliang Cheng", "Xiaowei Huang", "Qi Zhao" ], "kind": "evaluation-benchmark", "settings": [ "image-classification" ], "families": [], "applications": [ "robustness" ], "tier": "catalogued", "links": { "code": "https://github.com/zhouzhengqd/BEARD/", "project": "https://beard-leaderboard.github.io/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Introduces BEARD, an adversarial-robustness benchmark for dataset distillation covering DM, IDM and BACON distilled sets on CIFAR-10/100 and Tiny-ImageNet under FGSM/PGD/C&W attacks; defines a Robustness Ratio, an Attack Efficiency Ratio and a combined Comprehensive Robustness-Efficiency Index, and publishes results on a public leaderboard across several images-per-class settings and adversarial-training conditions.", "abstract": "Dataset Distillation (DD) is an emerging technique that compresses large-scale datasets into significantly smaller synthesized datasets while preserving high test performance and enabling the efficient training of large models. However, current research primarily focuses on enhancing evaluation accuracy under limited compression ratios, often overlooking critical security concerns such as adversarial robustness. A key challenge in evaluating this robustness lies in the complex interactions between distillation methods, model architectures, and adversarial attack strategies, which complicate standardized assessments. To address this, we introduce BEARD, an open and unified benchmark designed to systematically assess the adversarial robustness of DD methods, including DM, IDM, and BACON. BEARD encompasses a variety of adversarial attacks (e.g., FGSM, PGD, C&W) on distilled datasets like CIFAR-10/100 and TinyImageNet. Utilizing an adversarial game framework, it introduces three key metrics: Robustness Ratio (RR), Attack Efficiency Ratio (AE), and Comprehensive Robustness-Efficiency Index (CREI). Our analysis includes unified benchmarks, various Images Per Class (IPC) settings, and the effects of adversarial training. Results are available on the BEARD Leaderboard, along with a library providing model and dataset pools to support reproducible research. Access the code at BEARD.", "awesome_sections": [ "Main / Benchmark" ], "notes": "" }, { "id": "offline-behavior-distillation-2024", "type": "paper", "title": "Offline Behavior Distillation", "arxiv": "2410.22728", "url": null, "date": "2024-10-30", "venue": { "name": "NeurIPS 2024", "status": "conference" }, "authors": [ "Shiye Lei", "Sen Zhang", "Dacheng Tao" ], "kind": "application", "settings": [ "other-data" ], "families": [ "meta-learning-bptt" ], "applications": [ "rl" ], "tier": "notable", "links": { "code": "https://github.com/LeavesLei/OBD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "This paper formulates offline behavior distillation (OBD), synthesizing a small set of expert-like behavioral data from suboptimal offline RL data so that a policy can be learned quickly, and shows that naive decision-difference objectives are hard to optimize through the intractable bi-level problem and give only quadratic-discount-complexity guarantees. It introduces action-value weighted PBC (Av-PBC), proving an equivalence between policy performance and the action-value weighted decision difference, and reports faster convergence, better distillation performance and robust cross-architecture/optimizer generalization than the naive objectives on multiple D4RL datasets.", "abstract": "Massive reinforcement learning (RL) data are typically collected to train policies offline without the need for interactions, but the large data volume can cause training inefficiencies. To tackle this issue, we formulate offline behavior distillation (OBD), which synthesizes limited expert behavioral data from sub-optimal RL data, enabling rapid policy learning. We propose two naive OBD objectives, DBC and PBC, which measure distillation performance via the decision difference between policies trained on distilled data and either offline data or a near-expert policy. Due to intractable bi-level optimization, the OBD objective is difficult to minimize to small values, which deteriorates PBC by its distillation performance guarantee with quadratic discount complexity $\\mathcal{O}(1/(1-\\gamma)^2)$. We theoretically establish the equivalence between the policy performance and action-value weighted decision difference, and introduce action-value weighted PBC (Av-PBC) as a more effective OBD objective. By optimizing the weighted decision difference, Av-PBC achieves a superior distillation guarantee with linear discount complexity $\\mathcal{O}(1/(1-\\gamma))$. Extensive experiments on multiple D4RL datasets reveal that Av-PBC offers significant improvements in OBD performance, fast distillation convergence speed, and robust cross-architecture/optimizer generalization.", "notes": "" }, { "id": "less-is-more-efficient-2025", "type": "paper", "title": "Less is More: Efficient Time Series Dataset Condensation via Two-fold Modal Matching--Extended Version", "short": "TimeDC", "arxiv": "2410.20905", "url": null, "date": "2024-10-28", "venue": { "name": "VLDB 2025", "status": "journal" }, "authors": [ "Hao Miao", "Ziqiao Liu", "Yan Zhao", "Chenjuan Guo", "Bin Yang", "Kai Zheng", "Christian S. Jensen" ], "kind": "method", "settings": [ "time-series" ], "families": [ "trajectory-matching", "distribution-matching" ], "tier": "catalogued", "links": { "code": "https://github.com/uestc-liuzq/STdistillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "TimeDC condenses time series with two-fold modal matching, decomposition-driven frequency matching to preserve spectral structure and curriculum training-trajectory matching (with a buffer of precomputed expert trajectories) to preserve temporal dependencies, aiming for effective and generalized condensation rather than a single-domain surrogate objective. The paper reports effectiveness and efficiency gains over prior time-series condensation baselines across real datasets.", "abstract": "The expanding instrumentation of processes throughout society with sensors yields a proliferation of time series data that may in turn enable important applications, e.g., related to transportation infrastructures or power grids. Machine-learning based methods are increasingly being used to extract value from such data. We provide means of reducing the resulting considerable computational and data storage costs. We achieve this by providing means of condensing large time series datasets such that models trained on the condensed data achieve performance comparable to those trained on the original, large data. Specifically, we propose a time series dataset condensation framework, TimeDC, that employs two-fold modal matching, encompassing frequency matching and training trajectory matching. Thus, TimeDC performs time series feature extraction and decomposition-driven frequency matching to preserve complex temporal dependencies in the reduced time series. Further, TimeDC employs curriculum training trajectory matching to ensure effective and generalized time series dataset condensation. To avoid memory overflow and to reduce the cost of dataset condensation, the framework includes an expert buffer storing pre-computed expert trajectories. Extensive experiments on real data offer insight into the effectiveness and efficiency of the proposed solutions.", "notes": "The community name for this method is TimeDC (per its own abstract); note that the unrelated KDD 2024 paper timedc-2024 in this atlas calls itself CondTSC despite its id." }, { "id": "edf-2025", "type": "paper", "title": "Emphasizing Discriminative Features for Dataset Distillation in Complex Scenarios", "short": "EDF", "arxiv": "2410.17193", "url": null, "date": "2024-10-22", "venue": { "name": "CVPR 2025", "status": "conference" }, "authors": [ "Kai Wang", "Zekai Li", "Zhi-Qi Cheng", "Samir Khaki", "Ahmad Sajedi", "Ramakrishna Vedantam", "Konstantinos N Plataniotis", "Alexander Hauptmann", "Yang You" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/NUS-HPC-AI-Lab/EDF", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Uses Grad-CAM activation maps to up-weight discriminative image regions and down-weights low-loss (easy, redundant) supervision signals during synthesis, targeting the small, high-activation regions typical of complex datasets, and introduces the Comp-DD benchmark of sixteen ImageNet-1K subsets (eight easy, eight hard) to measure this, reporting consistent gains over prior methods on the hard subsets.", "abstract": "Dataset distillation has demonstrated strong performance on simple datasets like CIFAR, MNIST, and TinyImageNet but struggles to achieve similar results in more complex scenarios. In this paper, we propose EDF (emphasizes the discriminative features), a dataset distillation method that enhances key discriminative regions in synthetic images using Grad-CAM activation maps. Our approach is inspired by a key observation: in simple datasets, high-activation areas typically occupy most of the image, whereas in complex scenarios, the size of these areas is much smaller. Unlike previous methods that treat all pixels equally when synthesizing images, EDF uses Grad-CAM activation maps to enhance high-activation areas. From a supervision perspective, we downplay supervision signals that have lower losses, as they contain common patterns. Additionally, to help the DD community better explore complex scenarios, we build the Complex Dataset Distillation (Comp-DD) benchmark by meticulously selecting sixteen subsets, eight easy and eight hard, from ImageNet-1K. In particular, EDF consistently outperforms SOTA results in complex scenarios, such as ImageNet-1K subsets. Hopefully, more researchers will be inspired and encouraged to improve the practicality and efficacy of DD. Our code and benchmark will be made public at https://github.com/NUS-HPC-AI-Lab/EDF.", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ], "notes": "Family left as a generic optimization-recipe: the abstract does not name the matching family EDF is built on (NUS-HPC-AI-Lab has also published trajectory-matching work) — flagged for review." }, { "id": "lpld-2024", "type": "paper", "title": "Are Large-scale Soft Labels Necessary for Large-scale Dataset Distillation?", "short": "LPLD", "arxiv": "2410.15919", "url": null, "date": "2024-10-21", "venue": { "name": "NeurIPS 2024", "status": "conference" }, "authors": [ "Lingao Xiao", "Yang He" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "label-distillation" ], "tier": "notable", "links": { "code": "https://github.com/he-y/soft-label-pruning-for-dataset-distillation", "project": null }, "facets": { "labels": "soft-static", "regime": "decoupled", "prior": [ "pretrained-classifier" ], "scale": [ "imagenet-1k", "imagenet-21k", "tiny-imagenet" ] }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "SRe2L (2023) and CDA decouple synthesis from training via squeeze-recover-relabel: a teacher is pretrained once, images are synthesized by matching per-layer BatchNorm statistics, and every augmented crop is relabeled by the teacher at every training epoch. This relabel phase makes the stored soft-label set over 30x larger than the images themselves for ImageNet-1K. G-VBSM also re-batches within class but for a different reason (adding a model pool) and leaves BN statistics global. FKD (from model distillation) compresses labels by storing only the top-k logits.", "problem": "Nobody had asked why the relabel phase needs so much storage in the first place. The paper diagnoses the cause: SRe2L/CDA construct each optimization batch from different classes to match global BN statistics, so images belonging to the same class are optimized independently and end up nearly identical to each other (feature cosine similarity of 0.84 within class on ImageNet-1K, Table 1, versus 0.695 for real images) — this lack of within-class diversity is what forces per-crop, per-epoch relabeling to supply enough distinct supervision.", "idea": "If images within a class are made more diverse, the class no longer needs a fresh soft label for every augmentation and epoch; a large, redundant label pool can then be pruned by simple random sampling instead of a carefully engineered selection rule.", "method": "Modifies the SRe2L/CDA recover phase to batch and optimize images within one class together (rather than across classes) under class-wise BN running statistics computed in a one-epoch frozen-parameter pass during \"squeeze\" (class-wise supervision), increasing within-class feature diversity (Table 1: cosine similarity drops from 0.841/0.816 for SRe2L/CDA to 0.796 at IPC50) and lowering Maximum Mean Discrepancy to the real distribution. With images more diverse, the relabel-phase soft-label pool (one softmax vector per augmented crop per epoch, generated once by a pretrained ResNet-18/Timm teacher) is pruned by uniform random sampling at the batch level (\"improved label pool\") rather than any rule-based criterion (correctness, confidence, cut-ratio, etc., which Table 5 shows perform no better than random). In the loop: a pretrained teacher (frozen) for BN statistics and relabeling; no bi-level optimization or expert trajectories.", "evidence": "ImageNet-1K, ResNet-18 eval, teacher relabel-with-KD regime throughout (Table 3): at IPC200, unpruned (1x) SRe2L reaches 57.0%, CDA 63.3%, LPLD 62.6%; at 40x label pruning LPLD reaches 59.6% versus SRe2L's unpruned 57.0% and CDA's degraded score, i.e. LPLD with 40x less label storage still beats unpruned SRe2L by 2.6 points, corresponding to compressing labels from 113 GB to 2.8 GB. At IPC10, 1x LPLD is 34.6% vs SRe2L 20.1%/CDA 33.3%, and even at 40x LPLD (20.2%) beats unpruned SRe2L (20.1%). Tiny-ImageNet (Table 2) shows the same pattern, including a stated 7.5% accuracy gain over CDA at 40x label reduction. Ablation (Table 4, IPC50/ResNet18/ImageNet-1K) isolates each mechanism: class-wise batching alone (+C) raises 1x accuracy 52.0->54.7 but still collapses at 100x pruning (22.6, worse than baseline's 25.4); adding class-wise BN supervision (+CS) raises 100x accuracy to 29.1; adding the improved (batch-level) label pool (+ILP) raises it further to 33.7 — showing diversity alone is not sufficient without the improved pruning/sampling scheme. Compute: 4x A100 80GB GPUs for ImageNet-1K/21K-P, 1x for Tiny-ImageNet (Appendix E.5).", "limitations": "The paper states two limitations explicitly: generating the (large) soft-label pool is still required before pruning — pruning only reduces what is stored/reused, not the generation cost — and reducing label storage does not reduce training time, since the same number of training epochs is needed regardless of how many distinct labels are sampled. It only evaluates the decoupled BN-matching branch (SRe2L/CDA-style methods), not gradient/trajectory/distribution matching, and label-pruning quality is only validated at fairly high IPCs (10-200); scaling to larger networks (ResNet50/101) gives smaller or inconsistent gains than on ResNet18 (Table 2b)." }, "summary": "Traces the need for large stored soft-label sets in ImageNet-scale distillation to high within-class similarity, itself caused by batch-normalization matching that mixes samples from different classes in one batch; batching within classes during synthesis instead raises within-class diversity, letting soft labels be compressed with simple random pruning; compresses required soft labels from 113GB to 2.8GB (40x) at IPC=200 on ImageNet-1K with a 2.6-point accuracy gain.", "abstract": "In ImageNet-condensation, the storage for auxiliary soft labels exceeds that of the condensed dataset by over 30 times. However, are large-scale soft labels necessary for large-scale dataset distillation? In this paper, we first discover that the high within-class similarity in condensed datasets necessitates the use of large-scale soft labels. This high within-class similarity can be attributed to the fact that previous methods use samples from different classes to construct a single batch for batch normalization (BN) matching. To reduce the within-class similarity, we introduce class-wise supervision during the image synthesizing process by batching the samples within classes, instead of across classes. As a result, we can increase within-class diversity and reduce the size of required soft labels. A key benefit of improved image diversity is that soft label compression can be achieved through simple random pruning, eliminating the need for complex rule-based strategies. Experiments validate our discoveries. For example, when condensing ImageNet-1K to 200 images per class, our approach compresses the required soft labels from 113 GB to 2.8 GB (40x compression) with a 2.6% performance gain. Code is available at: https://github.com/he-y/soft-label-pruning-for-dataset-distillation", "builds_on": [ "sre2l-2023" ], "awesome_sections": [ "Main / Label Distillation" ] }, { "id": "teddy-2024", "type": "paper", "title": "Teddy: Efficient Large-Scale Dataset Distillation via Taylor-Approximated Matching", "short": "Teddy", "arxiv": "2410.07579", "url": null, "date": "2024-10-10", "venue": { "name": "ECCV 2024", "status": "conference" }, "authors": [ "Ruonan Yu", "Songhua Liu", "Jingwen Ye", "Xinchao Wang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes", "trajectory-matching" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Replaces the multi-step-gradient dependence of bi-level, trajectory-style dataset distillation with a memory-efficient first-order Taylor approximation, and replaces repeated per-iteration model training with a pre-cached pool of weak models generated from a single base model, reporting up to 12.8-point gains and 46.6% less runtime over prior methods on Tiny-ImageNet and full-size ImageNet-1K.", "abstract": "Dataset distillation or condensation refers to compressing a large-scale dataset into a much smaller one, enabling models trained on this synthetic dataset to generalize effectively on real data. Tackling this challenge, as defined, relies on a bi-level optimization algorithm: a novel model is trained in each iteration within a nested loop, with gradients propagated through an unrolled computation graph. However, this approach incurs high memory and time complexity, posing difficulties in scaling up to large datasets such as ImageNet. Addressing these concerns, this paper introduces Teddy, a Taylor-approximated dataset distillation framework designed to handle large-scale dataset and enhance efficiency. On the one hand, backed up by theoretical analysis, we propose a memory-efficient approximation derived from Taylor expansion, which transforms the original form dependent on multi-step gradients to a first-order one. On the other hand, rather than repeatedly training a novel model in each iteration, we unveil that employing a pre-cached pool of weak models, which can be generated from a single base model, enhances both time efficiency and performance concurrently, particularly when dealing with large-scale datasets. Extensive experiments demonstrate that the proposed Teddy attains state-of-the-art efficiency and performance on the Tiny-ImageNet and original-sized ImageNet-1K dataset, notably surpassing prior methods by up to 12.8%, while reducing 46.6% runtime. Our code will be available at https://github.com/Lexie-YU/Teddy.", "awesome_sections": [ "Main / Better Optimization" ] }, { "id": "mkdt-2025", "type": "paper", "title": "Dataset Distillation via Knowledge Distillation: Towards Efficient Self-Supervised Pre-Training of Deep Networks", "short": "MKDT", "arxiv": "2410.02116", "url": null, "date": "2024-10-03", "venue": { "name": "ICLR 2025", "status": "conference" }, "authors": [ "Siddharth Joshi", "Jiayi Ni", "Baharan Mirzasoleiman" ], "kind": "method", "settings": [ "self-supervised" ], "families": [ "trajectory-matching" ], "tier": "notable", "links": { "code": "https://github.com/jiayini1119/MKDT", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "MKDT shows that applying supervised trajectory matching directly to a self-supervised objective fails because the SSL gradient has high variance, and instead trains a small student to match the representations of a larger SSL-trained teacher (knowledge distillation), then distills a synthetic dataset by matching the training trajectories of these lower-variance student models. The paper reports up to 13 percentage points higher downstream accuracy than prior work when labeled data is limited, across a variety of transfer tasks, as the first effective dataset distillation method for self-supervised pre-training.", "abstract": "Dataset distillation (DD) generates small synthetic datasets that can efficiently train deep networks with a limited amount of memory and compute. Despite the success of DD methods for supervised learning, DD for self-supervised pre-training of deep models has remained unaddressed. Pre-training on unlabeled data is crucial for efficiently generalizing to downstream tasks with limited labeled data. In this work, we propose the first effective DD method for SSL pre-training. First, we show, theoretically and empirically, that naive application of supervised DD methods to SSL fails, due to the high variance of the SSL gradient. Then, we address this issue by relying on insights from knowledge distillation (KD) literature. Specifically, we train a small student model to match the representations of a larger teacher model trained with SSL. Then, we generate a small synthetic dataset by matching the training trajectories of the student models. As the KD objective has considerably lower variance than SSL, our approach can generate synthetic datasets that can successfully pre-train high-quality encoders. Through extensive experiments, we show that our distilled sets lead to up to 13% higher accuracy than prior work, on a variety of downstream tasks, in the presence of limited labeled data. Code at https://github.com/BigML-CS-UCLA/MKDT.", "notes": "" }, { "id": "drupi-2024", "type": "paper", "title": "DRUPI: Dataset Reduction Using Privileged Information", "short": "DRUPI", "arxiv": "2410.01611", "url": null, "date": "2024-10-02", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Shaobo Wang", "Youxin Jiang", "Tianle Niu", "Yantai Yang", "Ruiji Zhang", "Shuhao Hu", "Shuaiyu Zhang", "Chenghao Sun", "Weiya Li", "Conghui He", "Xuming Hu", "Linfeng Zhang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "label-distillation" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Introduces Dataset Condensation using Privileged Information (DRUPI/DCPI): alongside condensed images and labels, synthesizes auxiliary feature or attention labels as an additional training target; finds that moderately (not maximally) discriminative and diverse feature labels work best, and shows the technique plugs into existing condensation methods for consistent gains on ImageNet-1K, CIFAR-10/100 and Tiny-ImageNet.", "abstract": "Dataset Condensation (DC) seeks to select or distill samples from large datasets into smaller subsets while preserving performance on target tasks. Existing methods primarily focus on pruning or synthesizing data in the same format as the original dataset, typically being the input data and corresponding labels. However, in DC settings, we find it is possible to synthesize more information beyond the data-label pair as an additional learning target to facilitate model training. In this paper, we introduce Dataset Condensation using Privileged Information (DCPI), which enriches DC by synthesizing privileged information alongside the reduced dataset. This privileged information can take the form of feature labels or attention labels, providing auxiliary supervision to improve model learning. Our findings reveal that effective feature labels must balance between being overly discriminative and excessively diverse, with a moderate level proves optimal for improving the reduced dataset's efficacy. Extensive experiments on ImageNet-1K, CIFAR-10/100 and Tiny ImageNet demonstrate that DCPI integrates seamlessly with existing dataset condensation methods, offering significant performance gains.", "awesome_sections": [ "Main / Label Distillation" ] }, { "id": "dsdm-2024", "type": "paper", "title": "Diversified Semantic Distribution Matching for Dataset Distillation", "short": "DSDM", "arxiv": null, "url": "https://dl.acm.org/doi/10.1145/3664647.3680900", "date": "2024-10", "venue": { "name": "MM 2024", "status": "conference" }, "authors": [ "Hongcheng Li", "Yucan Zhou", "Xiaoyan Gu", "Bo Li", "Weiping Wang" ], "kind": "method", "settings": [ "image-classification", "other-data" ], "families": [ "distribution-matching" ], "tier": "catalogued", "links": { "code": "https://github.com/Li-Hongcheng/DSDM", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "acm", "depth": "abstract" }, "summary": "Matches both class-wise Gaussian prototypes and covariance matrices of pretrained-model features between real and synthetic data, rather than means alone, to increase within-class diversity of the distilled set, reporting state-of-the-art distribution-matching results on both image and speech datasets.", "abstract": null, "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ], "builds_on": [ "dm-2023" ] }, { "id": "dwa-2024", "type": "paper", "title": "Diversity-Driven Synthesis: Enhancing Dataset Distillation through Directed Weight Adjustment", "short": "DWA", "arxiv": "2409.17612", "url": null, "date": "2024-09-26", "venue": { "name": "NeurIPS 2024", "status": "conference" }, "authors": [ "Jiawei Du", "Xin Zhang", "Juncheng Hu", "Wenxin Huang", "Joey Tianyi Zhou" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "tier": "notable", "links": { "code": "https://github.com/AngusDujw/Diversity-Driven-Synthesis", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "labels": "soft-relabel", "prior": [ "pretrained-classifier" ], "regime": "decoupled", "parameterization": [ "pixels" ], "scale": [ "imagenet-1k", "tiny-imagenet", "mnist-cifar" ] }, "explained": { "before": "Builds directly on SRe2L's decoupled recovery loss (classification loss plus BN mean/variance matching). Does not name G-VBSM or CDA anywhere in the text; positions itself purely as a fix to SRe2L's per-image independent optimization, and separately criticizes MTT/TESLA (trajectory matching) for compute cost that blocks ImageNet-1K scaling.", "problem": "In decoupled synthesis, each synthetic image is optimized independently against the same fixed global teacher BN statistics, so images of one class end up similar. The paper proves the variance-matching term of the BN loss governs almost all of the diversity a synthetic batch can have, that random Gaussian initialization contributes only marginally, and that the mean- and variance-matching gradients can actively conflict, suppressing diversity further.", "idea": "Instead of adding random noise for diversity, perturb the teacher's weights in a directed way — chosen to maximize loss on a small random real-data batch, i.e. a sharpness-aware-style ascent step — so each synthetic image is optimized against a slightly different, data-grounded teacher. Because the perturbation is derived from real data rather than noise, it adds meaningful diversity while a first-order argument shows the perturbed teacher still generalizes.", "method": "Teacher: standard pretrained ResNet-18/50/101 (torchvision for ImageNet-1K; standard recipe for CIFAR/Tiny-ImageNet). Baseline recovery loss is SRe2L's, but with the BN coefficient split into separate $\\lambda_{mean}$ (fixed 0.01) and $\\lambda_{var}$ (tuned, optimum 0.11), since variance matching is shown to dominate diversity. Directed weight adjustment: $\\Delta\\tilde\\theta = \\arg\\max_{\\Delta\\theta} L_B(f_{\\theta_{\\mathcal T}+\\Delta\\theta})$ over a random real-data subset $B$, solved with $K{=}12$ forward-backward ascent steps and perturbation radius $\\rho{=}0.015$; the perturbed teacher is used for that image's recovery loss $\\ell(f_{\\theta_{\\mathcal T}+\\Delta\\tilde\\theta},s_i)+\\lambda_{mean}L_{mean}+\\lambda_{var}L_{var}$. No change to the relabeling step is described (no storage size or per-crop/stored distinction reported), and label regime is otherwise unchanged from SRe2L. Post-eval: ImageNet-1K 300 epochs, AdamW, cosine LR, RandomResizedCrop+Flip; CIFAR 400 epochs; Tiny-ImageNet 200 epochs, SGD.", "evidence": "Own re-run SRe2L baseline throughout (Table 1/2, ResNet-18): CIFAR-10 IPC=10 27.2% to DWA 32.6%, IPC=50 47.5% to 53.1%; CIFAR-100 IPC=10 31.6% to 39.6%, IPC=50 52.2% to 60.9%; Tiny-ImageNet IPC=50 41.1% to 52.8%; ImageNet-1K IPC=10 21.3% to 37.9% (+16.6 points), IPC=50 46.8% to 55.2% (+8.4 points), IPC=100 52.8% to 59.2%. Larger teacher/students at ImageNet-1K IPC=50: ResNet-50 55.6% to 62.3%, ResNet-101 60.8% to 63.3%. No hard-label result. Overhead: the abstract's \"<0.1%\" claim refers to marginal per-iteration cost, but Table 11 reports a measured 7.3% wall-clock overhead on CIFAR-100 (116.6s/IPC to 125.1s/IPC) — the two overhead figures in the paper are not consistent with each other. Key ablation (Table 3, CIFAR-100, ResNet-18): no adjustment IPC=10 30.6%/IPC=50 56.1%; random (undirected) $\\Delta\\theta$ IPC=10 collapses to 14.9%/IPC=50 56.2%; directed (DWA) IPC=10 39.6%/IPC=50 60.3% — showing direction, not magnitude of perturbation, is what helps, and that random perturbation actively hurts at low budgets.", "limitations": "States its own dependence on sampling a random distribution to select/scale the ascent step is a limitation: too much variance in that distribution can introduce unhelpful noise and bottleneck performance, as the random-perturbation ablation shows directly. No soft-label storage cost is reported despite the atlas-wide concern that relabeling dominates storage in this family. Proposes, but does not implement, sequential/curriculum synthesis for further diversity gains as future work." }, "builds_on": [ "sre2l-2023" ], "summary": "Provides a theoretical and empirical analysis of diversity within synthesized datasets and argues that the parallelizable but isolated synthesis common to decoupled methods needs an explicit diversity mechanism; introduces dynamic, directed weight adjustment during synthesis so each synthetic instance mirrors a large, varying subset of the real data, improving performance on CIFAR, Tiny-ImageNet and ImageNet-1K at minimal extra computational cost.", "abstract": "The sharp increase in data-related expenses has motivated research into condensing datasets while retaining the most informative features. Dataset distillation has thus recently come to the fore. This paradigm generates synthetic datasets that are representative enough to replace the original dataset in training a neural network. To avoid redundancy in these synthetic datasets, it is crucial that each element contains unique features and remains diverse from others during the synthesis stage. In this paper, we provide a thorough theoretical and empirical analysis of diversity within synthesized datasets. We argue that enhancing diversity can improve the parallelizable yet isolated synthesizing approach. Specifically, we introduce a novel method that employs dynamic and directed weight adjustment techniques to modulate the synthesis process, thereby maximizing the representativeness and diversity of each synthetic instance. Our method ensures that each batch of synthetic data mirrors the characteristics of a large, varying subset of the original dataset. Extensive experiments across multiple datasets, including CIFAR, Tiny-ImageNet, and ImageNet-1K, demonstrate the superior performance of our method, highlighting its effectiveness in producing diverse and representative synthetic datasets with minimal computational expense. Our code is available at https://github.com/AngusDujw/Diversity-Driven-Synthesis.https://github.com/AngusDujw/Diversity-Driven-Synthesis.", "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "dataset-distillation-based-hybrid-2024", "type": "paper", "title": "Dataset Distillation-based Hybrid Federated Learning on Non-IID Data", "short": "HFLDD", "arxiv": "2409.17517", "url": null, "date": "2024-09-26", "venue": { "name": "IEEE TNSE 2026", "status": "journal" }, "authors": [ "Xiufang Shi", "Wei Zhang", "Yuheng Li", "Mincheng Wu", "Zhenyu Wen", "Shibo He", "Tejal Shah", "Rajiv Ranjan" ], "kind": "application", "settings": [ "image-classification" ], "families": [], "applications": [ "federated-learning" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "For federated learning with skewed label distributions, groups clients into heterogeneous clusters whose combined labels are balanced, has cluster heads collect data distilled by their members, and trains with the server as if the pooled, distilled data were IID. Reports higher test accuracy and lower communication cost than baseline federated learning methods under severe label imbalance.", "abstract": "In federated learning, the heterogeneity of client data has a great impact on the performance of model training. Many heterogeneity issues in this process are raised by non-independently and identically distributed (non-IID) data. To address the issue of label distribution skew, we propose a hybrid federated learning framework called HFLDD, which integrates dataset distillation to generate approximately independent and equally distributed (IID) data, thereby improving the performance of model training. In particular, we partition the clients into heterogeneous clusters, where the data labels among different clients within a cluster are unbalanced while the data labels among different clusters are balanced. The cluster heads collect distilled data from the corresponding cluster members, and conduct model training in collaboration with the server. This training process is like traditional federated learning on IID data, and hence effectively alleviates the impact of non-IID data on model training. We perform a comprehensive analysis of the convergence behavior, communication overhead, and computational complexity of the proposed HFLDD. Extensive experimental results based on multiple public datasets demonstrate that when data labels are severely imbalanced, the proposed HFLDD outperforms the baseline methods in terms of both test accuracy and communication cost.", "notes": "Abstract does not name the underlying condensation mechanism used to distill each client's data; families left empty." }, { "id": "label-augmented-dataset-distillation-2025", "type": "paper", "title": "Label-Augmented Dataset Distillation", "arxiv": "2409.16239", "url": null, "date": "2024-09-24", "venue": { "name": "WACV 2025", "status": "conference" }, "authors": [ "Seoungyoon Kang", "Youngsun Lim", "Hyunjung Shim" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "label-distillation" ], "facets": { "labels": "soft-relabel" }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Sub-samples each synthetic image to generate additional dense labels that capture richer semantics than one label per image, at only about 2.5% extra storage on ImageNet subsets; the label-generation strategy is complementary to existing distillation methods, improving three high-performance algorithms by an average of 14.9% accuracy and improving cross-architecture robustness.", "abstract": "Traditional dataset distillation primarily focuses on image representation while often overlooking the important role of labels. In this study, we introduce Label-Augmented Dataset Distillation (LADD), a new dataset distillation framework enhancing dataset distillation with label augmentations. LADD sub-samples each synthetic image, generating additional dense labels to capture rich semantics. These dense labels require only a 2.5% increase in storage (ImageNet subsets) with significant performance benefits, providing strong learning signals. Our label generation strategy can complement existing dataset distillation methods for significantly enhancing their training efficiency and performance. Experimental results demonstrate that LADD outperforms existing methods in terms of computational overhead and accuracy. With three high-performance dataset distillation algorithms, LADD achieves remarkable gains by an average of 14.9% in accuracy. Furthermore, the effectiveness of our method is proven across various datasets, distillation hyperparameters, and algorithms. Finally, our method improves the cross-architecture robustness of the distilled dataset, which is important in the application scenario.", "awesome_sections": [ "Main / Label Distillation" ] }, { "id": "towards-model-agnostic-dataset-2024", "type": "paper", "title": "Towards Model-Agnostic Dataset Condensation by Heterogeneous Models", "arxiv": "2409.14538", "url": null, "date": "2024-09-22", "venue": { "name": "ECCV 2024", "status": "conference" }, "authors": [ "Jun-Yeong Moon", "Jung Uk Kim", "Gyeong-Moon Park" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/khu-agi/hmdc", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Addresses the tendency of condensed images to overfit to the particular network used during distillation by condensing with multiple heterogeneous models simultaneously, using a Gradient Balance Module to equalize each model's contribution and a Mutual Distillation with spatial-semantic decomposition to keep their semantics aligned, improving the cross-architecture usability of condensed images built on distribution-alignment and trajectory-tracking methods.", "abstract": "Abstract. The advancement of deep learning has coincided with the proliferation of both models and available data. The surge in dataset sizes and the subsequent surge in computational requirements have led to the development of the Dataset Condensation (DC). While prior studies have delved into generating synthetic images through methods like distribution alignment and training trajectory tracking for more efficient model training, a significant challenge arises when employing these condensed images practically. Notably, these condensed images tend to be specific to particular models, constraining their versatility and practicality. In response to this limitation, we introduce a novel method, Heterogeneous Model Dataset Condensation (HMDC), designed to produce universally applicable condensed images through cross-model interactions. To address the issues of gradient magnitude difference and semantic distance in models when utilizing heterogeneous models, we propose the Gradient Balance Module (GBM) and Mutual Distillation (MD) with the SpatialSemantic Decomposition method. By balancing the contribution of each model and maintaining their semantic meaning closely, our approach overcomes the limitations associated with model-specific condensed images and enhances the broader utility. The source code is available in https://github.com/KHU-AGI/HMDC.", "awesome_sections": [ "Main / Better Optimization" ] }, { "id": "dataset-distillation-from-first-2024", "type": "paper", "title": "Dataset Distillation from First Principles: Integrating Core Information Extraction and Purposeful Learning", "arxiv": "2409.01410", "url": null, "date": "2024-09-02", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Vyacheslav Kungurtsev", "Yuanfang Peng", "Jianyang Gu", "Saeed Vahidian", "Anthony Quinn", "Fadwa Idlahcen", "Yiran Chen" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Argues dataset distillation is under-specified without naming the downstream inference task, formalizes DD relative to a task-specific objective, and uses the formalization to analyze existing methods' faithfulness, with case studies on merging medical datasets that share only some features and on generating more physically faithful boundary data for physics-informed neural networks.", "abstract": "Dataset distillation (DD) is an increasingly important technique that focuses on constructing a synthetic dataset capable of capturing the core information in training data to achieve comparable performance in models trained on the latter. While DD has a wide range of applications, the theory supporting it is less well evolved. New methods of DD are compared on a common set of benchmarks, rather than oriented towards any particular learning task. In this work, we present a formal model of DD, arguing that a precise characterization of the underlying optimization problem must specify the inference task associated with the application of interest. Without this task-specific focus, the DD problem is under-specified, and the selection of a DD algorithm for a particular task is merely heuristic. Our formalization reveals novel applications of DD across different modeling environments. We analyze existing DD methods through this broader lens, highlighting their strengths and limitations in terms of accuracy and faithfulness to optimal DD operation. Finally, we present numerical results for two case studies important in contemporary settings. Firstly, we address a critical challenge in medical data analysis: merging the knowledge from different datasets composed of intersecting, but not identical, sets of features, in order to construct a larger dataset in what is usually a small sample setting. Secondly, we consider out-of-distribution error across boundary conditions for physics-informed neural networks (PINNs), showing the potential for DD to provide more physically faithful data. By establishing this general formulation of DD, we aim to establish a new research paradigm by which DD can be understood and from which new DD techniques can arise.", "awesome_sections": [ "Main / Better Understanding" ], "applications": [ "medical" ] }, { "id": "udd-2024", "type": "paper", "title": "UDD: Dataset Distillation via Mining Underutilized Regions", "short": "UDD", "arxiv": "2408.16268", "url": null, "date": "2024-08-29", "venue": { "name": "PRCV 2024", "status": "conference" }, "authors": [ "Shiguang Wang", "Zhongyu Zhang", "Jian Cheng" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization", "optimization-recipes" ], "facets": { "parameterization": [ "downsampled-multiformation" ] }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Identifies underutilized regions inside synthetic images as wasted capacity in multi-formation-style distillation, and introduces two utilization-sensitive search policies (response-based and data-jittering-based) that dynamically locate and fill these regions during optimization, together with a category-wise feature contrastive loss for inter-class separability. Reports 4.0% and 3.7% accuracy gains over the next-best method on CIFAR-10 and CIFAR-100 at IPC=1.", "abstract": "Dataset distillation synthesizes a small dataset such that a model trained on this set approximates the performance of the original dataset. Recent studies on dataset distillation focused primarily on the design of the optimization process, with methods such as gradient matching, feature alignment, and training trajectory matching. However, little attention has been given to the issue of underutilized regions in synthetic images. In this paper, we propose UDD, a novel approach to identify and exploit the underutilized regions to make them informative and discriminate, and thus improve the utilization of the synthetic dataset. Technically, UDD involves two underutilized regions searching policies for different conditions, i.e., response-based policy and data jittering-based policy. Compared with previous works, such two policies are utilization-sensitive, equipping with the ability to dynamically adjust the underutilized regions during the training process. Additionally, we analyze the current model optimization problem and design a category-wise feature contrastive loss, which can enhance the distinguishability of different categories and alleviate the shortcomings of the existing multi-formation methods. Experimentally, our method improves the utilization of the synthetic dataset and outperforms the state-of-the-art methods on various datasets, such as MNIST, FashionMNIST, SVHN, CIFAR-10, and CIFAR-100. For example, the improvements on CIFAR-10 and CIFAR-100 are 4.0\\% and 3.7\\% over the next best method with IPC=1, by mining the underutilized regions.", "notes": "" }, { "id": "nsd-2024", "type": "paper", "title": "Neural Spectral Decomposition for Dataset Distillation", "short": "NSD", "arxiv": "2408.16236", "url": null, "date": "2024-08-29", "venue": { "name": "ECCV 2024", "status": "conference" }, "authors": [ "Shaolei Yang", "Shen Cheng", "Mingbo Hong", "Haoqiang Fan", "Xing Wei", "Shuaicheng Liu" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization", "trajectory-matching" ], "tier": "catalogued", "links": { "code": "https://github.com/slyang2021/NSD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Represents the whole distilled dataset as a shared set of spectrum tensors combined pairwise with per-image transformation matrices, rather than as independent images, so information is shared across the synthetic set through simple matrix multiplication, and optimizes this low-rank representation with a trajectory-matching objective guided by the real distribution, reporting state-of-the-art results on CIFAR-10/100, Tiny-ImageNet and an ImageNet subset.", "abstract": "In this paper, we propose Neural Spectrum Decomposition, a generic decomposition framework for dataset distillation. Unlike previous methods, we consider the entire dataset as a high-dimensional observation that is low-rank across all dimensions. We aim to discover the low-rank representation of the entire dataset and perform distillation efficiently. Toward this end, we learn a set of spectrum tensors and transformation matrices, which, through simple matrix multiplication, reconstruct the data distribution. Specifically, a spectrum tensor can be mapped back to the image space by a transformation matrix, and efficient information sharing during the distillation learning process is achieved through pairwise combinations of different spectrum vectors and transformation matrices. Furthermore, we integrate a trajectory matching optimization method guided by a real distribution. Our experimental results demonstrate that our approach achieves state-of-the-art performance on benchmarks, including CIFAR10, CIFAR100, Tiny Imagenet, and ImageNet Subset. Our code are available at \\url{https://github.com/slyang2021/NSD}.", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ], "facets": { "parameterization": [ "factorized-bases" ] }, "builds_on": [ "mtt-2022" ] }, { "id": "distilling-long-tailed-datasets-2025", "type": "paper", "title": "Distilling Long-tailed Datasets", "short": "LTDD", "arxiv": "2408.14506", "url": null, "date": "2024-08-24", "venue": { "name": "CVPR 2025", "status": "conference" }, "authors": [ "Zhenghao Zhao", "Haoxuan Wang", "Yuzhang Shang", "Kai Wang", "Yan Yan" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "trajectory-matching" ], "facets": { "labels": "soft-static" }, "applications": [ "long-tail" ], "tier": "notable", "links": { "code": "https://github.com/ichbill/LTDD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Identifies why trajectory-matching distillation fails on long-tailed data — biased expert trajectories from imbalanced training propagate into biased synthetic sets, and experts trained on tail classes give poor guidance and soft labels — then proposes Distribution-agnostic Matching, which keeps the student trajectory away from the biased expert path, and Expert Decoupling, which matches backbone and classifier separately to improve tail-class guidance and soft-label quality. Reported as the first effective method for long-tailed dataset distillation.", "abstract": "Dataset distillation aims to synthesize a small, information-rich dataset from a large one for efficient model training. However, existing dataset distillation methods struggle with long-tailed datasets, which are prevalent in real-world scenarios. By investigating the reasons behind this unexpected result, we identified two main causes: 1) The distillation process on imbalanced datasets develops biased gradients, leading to the synthesis of similarly imbalanced distilled datasets. 2) The experts trained on such datasets perform suboptimally on tail classes, resulting in misguided distillation supervision and poor-quality soft-label initialization. To address these issues, we first propose Distribution-agnostic Matching to avoid directly matching the biased expert trajectories. It reduces the distance between the student and the biased expert trajectories and prevents the tail class bias from being distilled to the synthetic dataset. Moreover, we improve the distillation guidance with Expert Decoupling, which jointly matches the decoupled backbone and classifier to improve the tail class performance and initialize reliable soft labels. This work pioneers the field of long-tailed dataset distillation, marking the first effective effort to distill long-tailed datasets.", "awesome_sections": [ "Applications / Long-Tail" ], "notes": "" }, { "id": "not-all-samples-should-2025", "type": "paper", "title": "Not All Samples Should Be Utilized Equally: Towards Understanding and Improving Dataset Distillation", "arxiv": "2408.12483", "url": null, "date": "2024-08-22", "venue": { "name": "CVPR 2025 Workshop", "status": "workshop" }, "authors": [ "Shaobo Wang", "Yantai Yang", "Qilong Wang", "Kaixin Li", "Linfeng Zhang", "Junchi Yan" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Measures sample difficulty by gradient norm and finds that different matching-based distillation methods implicitly favor samples of different difficulty, then extends neural scaling laws for data pruning to dataset distillation to argue that prioritizing easier real samples during synthesis improves the distilled set, especially at low IPC, packaging this as a Sample Difficulty Correction plug-in that improves 7 distillation methods across 6 datasets.", "abstract": "Dataset Distillation (DD) aims to synthesize a small dataset capable of performing comparably to the original dataset. Despite the success of numerous DD methods, theoretical exploration of this area remains unaddressed. In this paper, we take an initial step towards understanding various matching-based DD methods from the perspective of sample difficulty. We begin by empirically examining sample difficulty, measured by gradient norm, and observe that different matching-based methods roughly correspond to specific difficulty tendencies. We then extend the neural scaling laws of data pruning to DD to theoretically explain these matching-based methods. Our findings suggest that prioritizing the synthesis of easier samples from the original dataset can enhance the quality of distilled datasets, especially in low IPC (image-per-class) settings. Based on our empirical observations and theoretical analysis, we introduce the Sample Difficulty Correction (SDC) approach, designed to predominantly generate easier samples to achieve higher dataset quality. Our SDC can be seamlessly integrated into existing methods as a plugin with minimal code adjustments. Experimental results demonstrate that adding SDC generates higher-quality distilled datasets across 7 distillation methods and 6 datasets.", "awesome_sections": [ "Main / Better Optimization" ] }, { "id": "dataset-distillation-for-histopathology-2024", "type": "paper", "title": "Dataset Distillation for Histopathology Image Classification", "short": "Histo-DD", "arxiv": "2408.09709", "url": null, "date": "2024-08-19", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Cong Cong", "Shiyu Xuan", "Sidong Liu", "Maurice Pagnucco", "Shiliang Zhang", "Yang Song" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "applications": [ "medical" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Adapts dataset distillation to histopathology whole-slide images by adding stain normalization and model augmentation to the distillation process, addressing the high color heterogeneity of histopathology patches. Reports more informative synthetic patches than coreset-selection and patch-sampling baselines on Camelyon16, TCGA-IDH and UniToPath at both patch- and slide-level classification, with architecture-agnostic synthetic samples.", "abstract": "Deep neural networks (DNNs) have exhibited remarkable success in the field of histopathology image analysis. On the other hand, the contemporary trend of employing large models and extensive datasets has underscored the significance of dataset distillation, which involves compressing large-scale datasets into a condensed set of synthetic samples, offering distinct advantages in improving training efficiency and streamlining downstream applications. In this work, we introduce a novel dataset distillation algorithm tailored for histopathology image datasets (Histo-DD), which integrates stain normalisation and model augmentation into the distillation progress. Such integration can substantially enhance the compatibility with histopathology images that are often characterised by high colour heterogeneity. We conduct a comprehensive evaluation of the effectiveness of the proposed algorithm and the generated histopathology samples in both patch-level and slide-level classification tasks. The experimental results, carried out on three publicly available WSI datasets, including Camelyon16, TCGA-IDH, and UniToPath, demonstrate that the proposed Histo-DD can generate more informative synthetic patches than previous coreset selection and patch sampling methods. Moreover, the synthetic samples can preserve discriminative information, substantially reduce training efforts, and exhibit architecture-agnostic properties. These advantages indicate that synthetic samples can serve as an alternative to large-scale datasets.", "awesome_sections": [ "Applications / Medical" ], "notes": "" }, { "id": "generative-dataset-distillation-based-2024", "type": "paper", "title": "Generative Dataset Distillation Based on Diffusion Model", "arxiv": "2408.08610", "url": null, "date": "2024-08-16", "venue": { "name": "ECCV 2024 Workshop", "status": "workshop" }, "authors": [ "Duo Su", "Junjie Hou", "Guang Li", "Ren Togo", "Rui Song", "Takahiro Ogawa", "Miki Haseyama" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "tier": "catalogued", "links": { "code": "https://github.com/Guang000/Generative-Dataset-Distillation-Based-on-Diffusion-Model", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Entry to the generative track of the ECCV 2024 Dataset Distillation Challenge: samples from SDXL-Turbo conditioned on class-name text prompts with post-generation augmentation, exploiting its speed to reach IPC=10 for Tiny-ImageNet and IPC=20 for CIFAR-100 within the challenge's 10-minute generation budget, versus IPC=1 for other diffusion entries; placed third in the track.", "abstract": "This paper presents our method for the generative track of The First Dataset Distillation Challenge at ECCV 2024. Since the diffusion model has become the mainstay of generative models because of its high-quality generative effects, we focus on distillation methods based on the diffusion model. Considering that the track can only generate a fixed number of images in 10 minutes using a generative model for CIFAR-100 and Tiny-ImageNet datasets, we need to use a generative model that can generate images at high speed. In this study, we proposed a novel generative dataset distillation method based on Stable Diffusion. Specifically, we use the SDXL-Turbo model which can generate images at high speed and quality. Compared to other diffusion models that can only generate images per class (IPC) = 1, our method can achieve an IPC = 10 for Tiny-ImageNet and an IPC = 20 for CIFAR-100, respectively. Additionally, to generate high-quality distilled datasets for CIFAR-100 and Tiny-ImageNet, we use the class information as text prompts and post data augmentation for the SDXL-Turbo model. Experimental results show the effectiveness of the proposed method, and we achieved third place in the generative track of the ECCV 2024 DD Challenge. Codes are available at https://github.com/Guang000/BANKO.", "awesome_sections": [ "Main / Generative Distillation / Diffusion" ] }, { "id": "helio-2025", "type": "paper", "title": "Heavy Labels Out! Dataset Distillation with Label Space Lightening", "short": "HeLlO", "arxiv": "2408.08201", "url": null, "date": "2024-08-15", "venue": { "name": "ICCV 2025", "status": "conference" }, "authors": [ "Ruonan Yu", "Songhua Liu", "Zigeng Chen", "Jingwen Ye", "Xinchao Wang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "label-distillation" ], "facets": { "labels": "soft-relabel", "regime": "decoupled", "prior": [ "vision-language-model" ], "scale": [ "imagenet-1k", "imagenet-subsets" ] }, "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "Decoupled ImageNet-scale methods (SRe2L, G-VBSM, RDED) all relabel every augmented crop at every training epoch with a pretrained teacher, and RDED's own numbers show why: on ImageNet-1K at IPC10, hard labels give 15.2% while soft labels give 42.1%. But storing those labels is expensive: the paper reports ImageNet-1K soft-label storage of 572 MB at IPC1 versus ~15 MB for the images (38x), rising to 110 GB at IPC200, comparable to the original dataset's size.", "problem": "Soft-label storage, not image storage, becomes the bottleneck of decoupled distillation at scale, yet no prior method addresses this — they treat the size of the teacher-generated label set as a fixed cost of the relabel phase.", "idea": "Replace stored labels with a small, fixed function that generates labels on demand: a CLIP-based image-to-label projector, fine-tuned cheaply with LoRA-style low-rank updates from an easily obtained \"weak teacher\" trajectory, can stand in for a fully pretrained teacher's soft labels at a tiny fraction of the storage, since only the low-rank adaptation matrices (not the labels themselves) need to be kept.", "method": "Builds a linear probe on frozen CLIP image features as the label projector; initializes the linear layer with CLIP's zero-shot text embeddings for the class names (proven equivalent to zero-shot classification, Proposition 1) so no extra parameters need to be stored for initialization. Fine-tunes the projector (CLIP image-encoder convolutions plus the linear head) with LoRA-style low-rank matrices (rank 8 for the image encoder, 64/128 for the linear head) using an MSE-plus-CE loss against soft labels from an ensemble of \"weak teacher\" checkpoints (9 checkpoints from a single ResNet-18 training trajectory, not a fully trained teacher). Synthetic images are initialized via RDED's patch-selection method, then further optimized to minimize the gap between the true teacher's and the projector's outputs on them. At evaluation, labels are generated online from the projector every step rather than stored; only the low-rank matrices (about 0.6-0.8M parameters) are kept, not any label tensor.", "evidence": "ImageNet-1K, ResNet-18 eval, all methods under teacher relabel-with-KD-style regimes (Table 1): HeLlO reaches 12.9% (IPC1, +6.3 over RDED's 6.6%), 43.7% (IPC10, +1.7 over RDED's 42.0%), and 52.2% (IPC50, versus RDED's 56.5% — a regression at the largest budget tested). Label storage for RDED/SRe2L-style relabeling on ImageNet-1K is 572.2 MB / 5722.0 MB / 28610.2 MB at IPC 1/10/50; HeLlO's projector needs roughly 1e-4x/1e-5x/3e-6x of that (its low-rank matrices, ~0.8M params, are fixed-size regardless of IPC), consistent with the abstract's \"0.003%\" headline figure at large scale. On ImageNet-100 HeLlO also improves accuracy at IPC1/10/50 (+4.4/+12.9/+7.8) while using #Params only 0.06x of RDED's teacher. Cross-architecture (Table 2, IPC10): HeLlO beats RDED on all 5 evaluation architectures, most on Swin-V2-Tiny (+11.7 on ImageNet-1K, +6.2 on ImageNet-100), suggesting the CLIP-based projector generalizes label quality better than a single-architecture teacher. Ablation (Table 3): starting from plain linear-probe CLIP (28.2% accuracy, ImageNet-100 IPC10) and adding multi-weak-teacher guidance (+1.9%), LoRA-like transfer (+13.4%, the largest single gain), text-embedding init (+0.1% but storage falls 1.5M->0.8M params), and image update (+0.1%) reaches 43.7%. Teacher-stage ablation (Table 4, right) shows earlier-stage (less converged) teachers help more at small IPC, later-stage teachers help more as IPC grows — echoing label-worth-2024's early-stopping finding in a different framework.", "limitations": "At the largest tested budget (ImageNet-1K IPC50) HeLlO's accuracy is lower than RDED's despite the huge storage saving, so the storage-accuracy trade-off is not uniformly favorable as scale grows. The method depends on a pretrained multimodal foundation model (CLIP) as the label-generation prior, which the atlas's facets record as an added prior not present in SRe2L/RDED; this ties accuracy to CLIP's own class-name/text-embedding coverage and may not transfer to domains poorly covered by CLIP's pretraining. Weak-teacher checkpoints are still needed for fine-tuning (though cheaper than a fully trained teacher). No wall-clock or GPU-hour cost of projector fine-tuning is reported, only storage and parameter counts. Evaluated only on ImageNet-100/1K with the RDED-style patch-selection initialization; no small-scale (CIFAR/TinyImageNet) or non-vision setting is tested." }, "summary": "Replaces stored per-crop soft labels with an image-to-label projector distilled from a foundation model (e.g. CLIP) via LoRA-style fine-tuning, so synthetic labels are generated online from synthetic images instead of stored, plus an image-optimization step that narrows the gap between original and distilled label generators; reports comparable accuracy to state-of-the-art distillation with only about 0.003% of the storage a full soft-label set would need on large-scale datasets.", "abstract": "Dataset distillation or condensation aims to condense a large-scale training dataset into a much smaller synthetic one such that the training performance of distilled and original sets on neural networks are similar. Although the number of training samples can be reduced substantially, current state-of-the-art methods heavily rely on enormous soft labels to achieve satisfactory performance. As a result, the required storage can be comparable even to original datasets, especially for large-scale ones. To solve this problem, instead of storing these heavy labels, we propose a novel label-lightening framework termed HeLlO aiming at effective image-to-label projectors, with which synthetic labels can be directly generated online from synthetic images. Specifically, to construct such projectors, we leverage prior knowledge in open-source foundation models, e.g., CLIP, and introduce a LoRA-like fine-tuning strategy to mitigate the gap between pre-trained and target distributions, so that original models for soft-label generation can be distilled into a group of low-rank matrices. Moreover, an effective image optimization method is proposed to further mitigate the potential error between the original and distilled label generators. Extensive experiments demonstrate that with only about 0.003% of the original storage required for a complete set of soft labels, we achieve comparable performance to current state-of-the-art dataset distillation methods on large-scale datasets. Our code will be available.", "awesome_sections": [ "Main / Label Distillation" ] }, { "id": "infer-2025", "type": "paper", "title": "Breaking Class Barriers: Efficient Dataset Distillation via Inter-Class Feature Compensator", "short": "INFER", "arxiv": "2408.06927", "url": null, "date": "2024-08-13", "venue": { "name": "ICLR 2025", "status": "conference" }, "authors": [ "Xin Zhang", "Jiawei Du", "Ping Liu", "Joey Tianyi Zhou" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis", "label-distillation" ], "tier": "notable", "links": { "code": "https://github.com/zhangxin-xd/UFC", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "labels": "mixed", "prior": [ "pretrained-classifier" ], "regime": "decoupled", "parameterization": [ "pixels" ], "scale": [ "imagenet-1k", "tiny-imagenet", "mnist-cifar" ] }, "explained": { "before": "Builds on SRe2L's decoupled BN-statistics-plus-logit recovery and its dynamic, per-crop per-epoch soft-label relabeling via MixUp, which the paper singles out for its storage cost (25.9GB of labels versus 0.7GB of images on ImageNet-1K).", "problem": "Every synthetic instance in prior class-specific frameworks is tied to one pre-assigned one-hot label and optimized only to encode that class's intra-class features. As IPC grows, new instances increasingly duplicate existing intra-class features rather than adding information, and no instance is ever optimized to represent the inter-class feature regions that actually shape decision boundaries.", "idea": "Learn a single additive Universal Feature Compensator (UFC) — a zero-initialized, image-shaped tensor optimized jointly across all classes in a subset — and add it to one real image per class to generate a full new synthetic instance per class from one UFC. Because the resulting instances already span multiple classes, their soft labels behave like MixUp-interpolated labels, so a single static soft label per instance can replace SRe2L's dynamically regenerated, stored-every-epoch labels.", "method": "Teacher ensemble (also used for relabeling): ResNet-18, MobileNetV2, EfficientNet-B0 (+ShuffleNetV2 for CIFAR/Tiny-ImageNet). The base set is one unoptimized real image per class; the only optimized quantity is the additive compensator $u_j$, trained with the same BN-statistics-plus-classification recovery loss as SRe2L on $\\tilde s_i=x_i+u_j$, so $M$ compensators over $C$ real images yield $C\\times M$ synthetic instances. Soft labels are computed once per instance as the teacher ensemble's average prediction $\\tilde y_i=\\frac1M\\sum_m f^m_{\\theta_{\\mathcal T}}(\\tilde s_i)$ and stored statically; MixUp at student-training time linearly combines these stored labels rather than recomputing them, cutting label storage up to 99.3%. Budget/compression is reported both as IPC (counting UFC-generated instances) and as a compression ratio that includes the UFC's own storage bits, so comparisons to SRe2L are at equal total stored bits, not equal image count. Post-eval: AdamW, 300 epochs (static labels) or 75 epochs (dynamic-label variant, reduced 4x for fair comparison), RandomResizedCrop+Flip+MixUp; students ResNet-18/50/101 for ImageNet-1K, plus MobileNetV2/EfficientNet-B0/ShuffleNetV2 for CIFAR-100 cross-architecture tests.", "evidence": "ImageNet-1K, ResNet-18 (Table 2): IPC=10, INFER 37.0% vs SRe2L 21.3%; IPC=50, 54.3% vs 46.8% (a 7.5 point gain at IPC=50, ResNet-18 — the abstract's \"34.5% at IPC=50\" figure does not correspond to any ResNet-18 or ResNet-101 cell found in the paper's own tables and could not be reconciled). On ResNet-50, IPC=50: 62.5% vs 55.6%. CIFAR-100, ResNet-18: IPC=10, INFER 50.2% vs SRe2L 31.6%; IPC=50, 65.1% vs 49.5% (a 15.6-point gain, and with dynamic labels, INFER+Dyn reaches 68.9%, a 19.4-point gain). Storage: SRe2L needs 25.9GB of dynamic soft labels on ImageNet-1K versus 0.7GB of images; INFER's static labels cut this by up to 99.3%. Key ablation (Table 4, ImageNet-1K IPC=10): adding architectures to the UFC-optimization ensemble raises INFER from 45.0% (ResNet-18 only) to 50.2% (four architectures), and INFER+Dyn from 38.1% to 53.4%, isolating ensemble diversity as the mechanism, not just the UFC idea itself. No hard-label result is reported.", "limitations": "States the single-UFC-per-subset design likely will not scale to ImageNet-21K-sized label spaces since one compensator must represent generality across all classes in the subset, and that its two-level structure (real per-class images for intra-class, UFC for inter-class) is not jointly optimized, left as future work. Compression-ratio accounting that folds in UFC storage is reported only for ImageNet-1K, not for the CIFAR/Tiny-ImageNet tables, leaving the fairness of those comparisons less transparent." }, "builds_on": [ "sre2l-2023" ], "summary": "Breaks the one-hot, per-class synthesis barrier of prior distillation with a Universal Feature Compensator (UFC) that enhances feature integration across classes and can generate multiple additional synthetic instances from a single UFC input, plus label interpolation across classes that shrinks stored soft-label size toward zero; reports outperforming SRe2L by 34.5% at IPC=50 on ImageNet-1K with ResNet-18.", "abstract": "Dataset distillation has emerged as a technique aiming to condense informative features from large, natural datasets into a compact and synthetic form. While recent advancements have refined this technique, its performance is bottlenecked by the prevailing class-specific synthesis paradigm. Under this paradigm, synthetic data is optimized exclusively for a pre-assigned one-hot label, creating an implicit class barrier in feature condensation. This leads to inefficient utilization of the distillation budget and oversight of inter-class feature distributions, which ultimately limits the effectiveness and efficiency, as demonstrated in our analysis. To overcome these constraints, this paper presents the Inter-class Feature Compensator (INFER), an innovative distillation approach that transcends the class-specific data-label framework widely utilized in current dataset distillation methods. Specifically, INFER leverages a Universal Feature Compensator (UFC) to enhance feature integration across classes, enabling the generation of multiple additional synthetic instances from a single UFC input. This significantly improves the efficiency of the distillation budget. Moreover, INFER enriches inter-class interactions during the distillation, thereby enhancing the effectiveness and generalizability of the distilled data. By allowing for the linear interpolation of labels similar to those in the original dataset, INFER meticulously optimizes the synthetic data and dramatically reduces the size of soft labels in the synthetic dataset to almost zero, establishing a new benchmark for efficiency and effectiveness in dataset distillation. In practice, INFER demonstrates state-of-the-art performance across benchmark datasets. For instance, in the ipc = 50 setting on ImageNet-1k with the same compression level, it outperforms SRe2L by 34.5% using ResNet18.", "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "pad-2024", "type": "paper", "title": "Prioritize Alignment in Dataset Distillation", "short": "PAD", "arxiv": "2408.03360", "url": null, "date": "2024-08-06", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Zekai Li", "Ziyao Guo", "Wangbo Zhao", "Tianle Zhang", "Zhi-Qi Cheng", "Samir Khaki", "Kaipeng Zhang", "Ahmad Sajedi", "Konstantinos N Plataniotis", "Kai Wang", "Yang You" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "trajectory-matching" ], "facets": { "parameterization": [ "pixels" ], "labels": "learned", "prior": [ "trained-experts" ], "regime": "single-level", "scale": [ "tiny-imagenet" ] }, "tier": "notable", "links": { "code": "https://github.com/NUS-HPC-AI-Lab/PAD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Identifies that trajectory-matching distillation extracts and embeds 'misaligned' information from its agent (expert) model, and fixes this by pruning the target dataset to match the compression ratio before computing expert trajectories and by using only the agent model's deep layers for the matching loss, avoiding excess low-level information; reports state-of-the-art results built on trajectory matching.", "abstract": "Dataset Distillation aims to compress a large dataset into a significantly more compact, synthetic one without compromising the performance of the trained models. To achieve this, existing methods use the agent model to extract information from the target dataset and embed it into the distilled dataset. Consequently, the quality of extracted and embedded information determines the quality of the distilled dataset. In this work, we find that existing methods introduce misaligned information in both information extraction and embedding stages. To alleviate this, we propose Prioritize Alignment in Dataset Distillation (PAD), which aligns information from the following two perspectives. 1) We prune the target dataset according to the compressing ratio to filter the information that can be extracted by the agent model. 2) We use only deep layers of the agent model to perform the distillation to avoid excessively introducing low-level information. This simple strategy effectively filters out misaligned information and brings non-trivial improvement for mainstream matching-based distillation algorithms. Furthermore, built on trajectory matching, \\textbf{PAD} achieves remarkable improvements on various benchmarks, achieving state-of-the-art performance.", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ], "builds_on": [ "mtt-2022", "datm-2024" ], "explained": { "before": "Built directly on datm-2024 as its backbone (same distillation/evaluation configuration, including datm-2024's learned soft labels and difficulty-aligned trajectory segments), and responds more broadly to mtt-2022-style trajectory matching, framing both gradient- and trajectory-matching methods as sharing a common flaw regardless of which segment or expert they match.", "problem": "Argues that matching-based distillation methods extract and embed \"misaligned information\" at two separate points in the pipeline, independent of which trajectory segment is matched: (1) extraction -- the agent (expert) model is trained on the full real dataset even though the synthetic set will only ever need to represent a small compression-ratio-sized slice of it, so some of what the agent learns is never usable by the synthetic data; (2) embedding -- matching shallow-layer agent parameters injects low-level, quickly-saturated texture information that a small synthetic set cannot usefully represent, crowding out the higher-level information deep layers carry.", "idea": "Align what the agent model is allowed to extract and what gets embedded in the matching loss with the actual compression ratio: prune the real target dataset (removing the easiest, least-informative samples) before training experts, so experts only ever see data at roughly the same \"difficulty density\" the synthetic set will need to represent; and mask out shallow-layer agent parameters during matching, using only deep-layer parameters, since shallow layers were empirically found to carry redundant, low-level signal that saturates almost immediately.", "method": "Two add-on modules layered on datm-2024's trajectory-matching pipeline (same synthetic images, learned soft labels, and difficulty-aligned matching window): Filtered Information EXtraction (FIEX) removes a tuned ratio of the easiest real training samples before expert trajectories are trained, so experts are trained on a pruned, harder target set matched to the compression ratio; Filtered Information EMbedding (FIEM) adds a parameter-selection mask that excludes a ratio of shallow-layer agent parameters (ordered shallow-to-deep) from the trajectory-matching distance, so only deeper-layer parameters contribute to the loss. Both ratios are tuned hyperparameters. What is stored: identical to datm-2024 -- per-class synthetic images (pixels) plus learned soft labels; expert trajectories remain a synthesis-time cache, now trained on pruned data.", "evidence": "Table 4 (ConvNet, learned soft labels, DATM value in parens): CIFAR-10 IPC1/10/50/500/1000 = 47.2+/-0.6 (46.9) / 67.4+/-0.3 (66.8) / 77.0+/-0.5 (76.1) / 84.6+/-0.3 (83.5) / 86.7+/-0.2% (85.5) vs. full-data 84.8+/-0.1%; CIFAR-100 IPC1/10/50/100 = 28.4+/-0.5 (27.9) / 47.8+/-0.2 (47.2) / 55.9+/-0.3 (55.0) / 58.5+/-0.3% (57.5) vs. full-data 56.2+/-0.3%; Tiny-ImageNet IPC1/10/50 = 17.7+/-0.2 (17.1) / 32.3+/-0.4 (31.1) / 41.6+/-0.4% (39.7) vs. full-data 37.6+/-0.4% -- PAD beats datm-2024 at every IPC in this table, by margins of roughly 0.3-1.9 points. Cross-architecture (CIFAR-10 IPC50): PAD 55.91/52.35/44.97/45.92% on ConvNet/ResNet18/VGG/AlexNet vs. DATM 55.03/51.71/45.38/45.74% -- PAD slightly ahead on ConvNet/ResNet18 but marginally behind DATM on VGG/AlexNet. Against RDED (Table 8, a decoupled-synthesis/realistic-selection method, outside this family): PAD wins CIFAR-10 IPC1/50/500 (26.8/62.3/89.6% vs. RDED's 23.5/50.2/87.2%) and CIFAR-100 IPC10/50 (50.2/59.3% vs. 48.1/57.0%), but *loses* to RDED on CIFAR-100 IPC1 (16.4% vs. 19.6%) and on Tiny-ImageNet IPC1/10 (10.1/37.2% vs. 12.0/39.6%) -- direct evidence that a decoupled-synthesis method beats the trajectory-matching family's best result at exactly the harder, more-classes end of the CIFAR/Tiny-ImageNet range. Cost: run on 4x NVIDIA A100 (80GB each), the same hardware as datm-2024. Ablation (Table 3, two benchmarks): neither FIEX nor FIEM alone reaches the full method -- e.g. baseline 66.7%, +FIEM 66.9%, +FIEX 67.2%, both 67.4% (one benchmark), and baseline 55.0%, +FIEM 55.5%, +FIEX 55.8%, both 56.2% (another) -- isolating extraction-stage and embedding-stage misalignment as two separate, additive, real problems, with FIEX (data pruning) contributing somewhat more than FIEM (layer masking) in both cases. A separate comparison shows FIEX beats BLiP (a prior data-utility-based pruning method) at matched IPC.", "limitations": "The FIEX pruning ratio and FIEM shallow-layer-masking ratio are additional hyperparameters tied to the specific compression ratio being targeted, tuned per dataset/IPC rather than derived analytically. Inherits datm-2024's full cost and label-regime profile: the same cached-expert-trajectory infrastructure, 4x 80GB-A100 compute, and learned soft labels (so PAD's numbers are not comparable to any hard-label trajectory-matching result). No ImageNet-1K evaluation. Against RDED, a much cheaper decoupled-synthesis method outside this family, PAD's advantage is inconsistent -- it loses on CIFAR-100 IPC1 and Tiny-ImageNet IPC1/10, suggesting the alignment fix narrows but does not close the gap to non-bilevel, teacher-driven synthesis at the harder end of the benchmark range." } }, { "id": "one-shot-collaborative-data-2024", "type": "paper", "title": "One-Shot Collaborative Data Distillation", "short": "CollabDM", "arxiv": "2408.02266", "url": null, "date": "2024-08-05", "venue": { "name": "ECAI 2024", "status": "conference" }, "authors": [ "William Holland", "Chandra Thapa", "Sarah Ali Siddiqui", "Wei Shao", "Seyit Camtepe" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "applications": [ "federated-learning" ], "tier": "catalogued", "links": { "code": "https://github.com/rayneholland/CollabDM", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "CollabDM captures the global data distribution across distributed clients in a single round of client-server communication, rather than merging independently-distilled local sets (which the paper shows degrades under client data heterogeneity). Reports outperforming the state-of-the-art one-shot distillation method on skewed distributed data and demonstrates benefit on 5G network attack detection.", "abstract": "Large machine-learning training datasets can be distilled into small collections of informative synthetic data samples. These synthetic sets support efficient model learning and reduce the communication cost of data sharing. Thus, high-fidelity distilled data can support the efficient deployment of machine learning applications in distributed network environments. A naive way to construct a synthetic set in a distributed environment is to allow each client to perform local data distillation and to merge local distillations at a central server. However, the quality of the resulting set is impaired by heterogeneity in the distributions of the local data held by clients. To overcome this challenge, we introduce the first collaborative data distillation technique, called CollabDM, which captures the global distribution of the data and requires only a single round of communication between client and server. Our method outperforms the state-of-the-art one-shot learning method on skewed data in distributed learning environments. We also show the promising practical benefits of our method when applied to attack detection in 5G networks.", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "" }, { "id": "dataset-distillation-for-offline-2024", "type": "paper", "title": "Dataset Distillation for Offline Reinforcement Learning", "arxiv": "2407.20299", "url": null, "date": "2024-07-29", "venue": { "name": "ICML 2024 Workshop", "status": "workshop" }, "authors": [ "Jonathan Light", "Yuanzhe Liu", "Ziniu Hu" ], "kind": "application", "settings": [ "other-data" ], "families": [], "applications": [ "rl" ], "tier": "catalogued", "links": { "code": "https://github.com/ggflow123/DDRL", "project": "https://datasetdistillation4rl.github.io/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Applies dataset distillation to offline reinforcement learning, distilling a smaller synthetic dataset (DDRL) from offline trajectories so that a policy trained on it matches one trained on the full offline dataset or with percentile behavioral cloning.", "abstract": "Offline reinforcement learning often requires a quality dataset that we can train a policy on. However, in many situations, it is not possible to get such a dataset, nor is it easy to train a policy to perform well in the actual environment given the offline data. We propose using data distillation to train and distill a better dataset which can then be used for training a better policy model. We show that our method is able to synthesize a dataset where a model trained on it achieves similar performance to a model trained on the full dataset or a model trained using percentile behavioral cloning. Our project site is available at https://datasetdistillation4rl.github.io . We also provide our implementation at https://github.com/ggflow123/DDRL .", "awesome_sections": [ "Applications / Reinforcement Learning" ], "notes": "Abstract does not specify the underlying matching mechanism; family left empty." }, { "id": "d4m-2024", "type": "paper", "title": "D$^4$M: Dataset Distillation via Disentangled Diffusion Model", "short": "D4M", "arxiv": "2407.15138", "url": null, "date": "2024-07-21", "venue": { "name": "CVPR 2024", "status": "conference" }, "authors": [ "Duo Su", "Junjie Hou", "Weizhi Gao", "Yingjie Tian", "Bowen Tang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "labels": "soft-relabel", "prior": [ "text-to-image-diffusion" ], "regime": "training-free", "scale": [ "imagenet-1k" ] }, "tier": "landmark", "links": { "code": "https://github.com/suduo94/D4M", "project": "https://junjie31.github.io/D4M/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Argues that architecture-dependent bi-level/uni-level matching objectives are the reason cross-architecture generalization degrades, and instead condenses the dataset into a latent diffusion model's space, incorporating label information into per-category prototypes so the same distilled representation serves any evaluation architecture without regenerating separate datasets; reports superior performance and robust generalization across most benchmarks compared with prior state-of-the-art methods.", "abstract": "Dataset distillation offers a lightweight synthetic dataset for fast network training with promising test accuracy. To imitate the performance of the original dataset, most approaches employ bi-level optimization and the distillation space relies on the matching architecture. Nevertheless, these approaches either suffer significant computational costs on large-scale datasets or experience performance decline on cross-architectures. We advocate for designing an economical dataset distillation framework that is independent of the matching architectures. With empirical observations, we argue that constraining the consistency of the real and synthetic image spaces will enhance the cross-architecture generalization. Motivated by this, we introduce Dataset Distillation via Disentangled Diffusion Model (D$^4$M), an efficient framework for dataset distillation. Compared to architecture-dependent methods, D$^4$M employs latent diffusion model to guarantee consistency and incorporates label information into category prototypes. The distilled datasets are versatile, eliminating the need for repeated generation of distinct datasets for various architectures. Through comprehensive experiments, D$^4$M demonstrates superior performance and robust generalization, surpassing the SOTA methods across most aspects.", "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "explained": { "before": "Bi-level (STM, synthesis-time matching: DC, DSA, CAFE, MTT, TESLA) and SRe2L's dual-time-matching (DTM: squeeze-recover-relabel) methods both couple image synthesis to a specific network architecture — STM matches gradients/features/trajectories against one architecture during synthesis, DTM recovers images from a fixed teacher's BN statistics — so both must regenerate a dataset (or at least verify it) for each new evaluation architecture, and STM in particular becomes computationally infeasible or inaccurate at ImageNet-1K scale. GLaD constrains the output to a GAN's manifold but the inner loop still matches through a specific network.", "problem": "Every synthesis-time matching approach biases the synthetic image space toward whatever architecture supplies the loss, which the paper argues both caps cross-architecture generalization and produces images with degraded human-legible semantics (Fig. 2); STM also scales matching cost/GPU-memory with the chosen architecture's size (MTT needs 79.9GB on ImageNet-1K at 128x128 over one V100-class setup), making ImageNet-1K distillation impractical for the STM paradigm.", "idea": "Remove architecture matching from the synthesis step entirely: extract per-category prototypes (cluster centroids) from a frozen, off-the-shelf latent diffusion model's own latent space, condition the frozen diffusion model on the prototype + a text-encoded label to generate images, and push all learning into Training-Time Matching (TTM) — i.e. soft-label relabeling during downstream network training — so no architecture ever touches the synthesis process.", "method": "A frozen, publicly released Stable Diffusion V1-5 checkpoint (a general text-to-image LDM pretrained on LAION-scale web image-text pairs, not on the target classification dataset) supplies encoder $\\mathcal{E}$, decoder $\\mathcal{D}$, text encoder $\\tau_\\theta$, and denoising U-Net $\\mathcal{U}_t$, all frozen. Real images per category are encoded to latents $z=\\mathcal{E}(\\mathcal{T})$, then Mini-Batch k-Means clusters them into $C$ prototypes $z^c$ per class (for IPC<100, $C$ is set equal to IPC; for larger IPC, e.g. 10-50 prototypes with multiple samples generated per prototype). Each prototype is noised via the forward process to a chosen strength (0.7, an img2img-style partial-noising parameter) and denoised while cross-attending to the label's text embedding $\\tau_\\theta(L)$ with classifier-free guidance scale 8, then decoded: $\\mathcal{S}=\\mathcal{D}(\\tilde{Z}^c)$. No gradients flow through the diffusion model at any point (synthesis is training-free). At evaluation, a teacher network (ResNet-18/50/101 pretrained on the real dataset) generates soft labels for the distilled images at every training epoch (an FKD-style scheme, Eq. 8) and the student is trained with KL- divergence to these soft predictions rather than hard one-hot labels — this soft-relabeling (TTM) is presented as itself a distribution-matching mechanism, distinct from the synthesis step.", "evidence": "ImageNet-1K, IPC=100, ResNet-18 teacher/student, soft-relabel TTM (Table 2): D4M 66.5% vs. SRe2L 52.8% vs. full-dataset ceiling 69.8%; IPC=10: D4M 27.9% vs. SRe2L 21.3% vs. TESLA (STM, ConvNetD4 teacher) 7.7%. Tiny-ImageNet IPC=100: D4M 51.4% vs. SRe2L 50.8% (R18); a \"D4M-G\" variant, built by simply extracting 200 of the already-distilled ImageNet-1K classes rather than distilling Tiny-ImageNet from scratch, nearly matches or exceeds dedicated D4M (53.3% R18), evidencing the claimed architecture/ dataset-versatility. CIFAR-10/100 (Table 1, small-scale, STM baselines KIP/FRePO/DSA/CAFE/TESLA copied from originals): D4M beats FRePo and TESLA on CIFAR-100 IPC=10 by 2.5/3.3 points (45.0% vs. 42.5%/41.7%) but underperforms TESLA on CIFAR-10 IPC=10 (56.2% vs. 66.4%), and the paper itself notes STM still wins at small category counts/IPC, TTM only becoming favorable as classes grow. Cost (Table 6): on ImageNet- 1K, D4M synthesis is 2.7s/image at 6.1GB GPU memory (fixed, architecture-free) vs. SRe2L 5.2s/34.8GB and MTT 45.0s/79.9GB (both architecture-dependent and rising with matching-network size) — D4M is 3.82x faster than SRe2L end-to-end per the paper's own count. Key ablations: (1) Table 3 — applying an additional STM step (BN-distribution matching) on top of D4M's already-generated images *reduces* accuracy at every teacher (e.g. R18 teacher: 27.9%->23.6%, a 4.3-point drop), because it disrupts the fused image+text information already encoded by the diffusion conditioning — architecture matching is actively harmful once TTM is in place, not merely unnecessary; (2) Table 4 — replacing prototype initialization with random noise drops ImageNet-1K IPC=10 accuracy from 27.9% to 15.6% (a 12.3-point gap), confirming prototypes (not just \"any diffusion sample\") are the mechanism.", "limitations": "The diffusion model (Stable Diffusion V1-5) is a general, off-the-shelf text-to-image model trained on web-scale LAION data, never fine-tuned or trained on CIFAR/ImageNet/Tiny-ImageNet — this is the clearest case in the group of a generator with zero target-dataset training, but also means all reported gains are entangled with soft-label relabeling (TTM) at evaluation, so the paper does not isolate how much of its advantage over SRe2L comes from image quality/diversity versus from label regime (both differ simultaneously in every ImageNet-1K/Tiny-ImageNet comparison). The paper's own stated limitation is a \"significant performance degradation\" at extreme low budgets (IPC=1/10), where CIFAR-10 IPC=10 D4M (56.2%) trails TESLA (66.4%) by 10 points despite winning at larger IPC/more classes; text prompts are only class-name-level (no richer per-image captions, unlike later VLM-prompted successors); the strength/guidance-scale hyperparameters (0.7/8) are tuned on ImageNet-1K/ResNet and not shown to transfer automatically to other domains." } }, { "id": "dataset-distillation-in-medical-2024", "type": "paper", "title": "Dataset Distillation in Medical Imaging: A Feasibility Study", "arxiv": "2407.14429", "url": null, "date": "2024-07-19", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Muyang Li", "Can Cui", "Quan Liu", "Ruining Deng", "Tianyuan Yao", "Marilyn Lionts", "Yuankai Huo" ], "kind": "evaluation-benchmark", "settings": [ "image-classification" ], "families": [], "applications": [ "medical" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Empirically tests whether existing dataset distillation methods, developed for natural images, transfer to medical imaging, across datasets with minor and major domain variation, and asks whether a small representative sample can predict distillation success. Reports that distillation can substantially reduce medical dataset size while keeping performance comparable to full-data training, and that a small representative image sample is a reliable indicator of how well distillation will work on a given dataset.", "abstract": "Data sharing in the medical image analysis field has potential yet remains underappreciated. The aim is often to share datasets efficiently with other sites to train models effectively. One possible solution is to avoid transferring the entire dataset while still achieving similar model performance. Recent progress in data distillation within computer science offers promising prospects for sharing medical data efficiently without significantly compromising model effectiveness. However, it remains uncertain whether these methods would be applicable to medical imaging, since medical and natural images are distinct fields. Moreover, it is intriguing to consider what level of performance could be achieved with these methods. To answer these questions, we conduct investigations on a variety of leading data distillation methods, in different contexts of medical imaging. We evaluate the feasibility of these methods with extensive experiments in two aspects: 1) Assess the impact of data distillation across multiple datasets characterized by minor or great variations. 2) Explore the indicator to predict the distillation performance. Our extensive experiments across multiple medical datasets reveal that data distillation can significantly reduce dataset size while maintaining comparable model performance to that achieved with the full dataset, suggesting that a small, representative sample of images can serve as a reliable indicator of distillation success. This study demonstrates that data distillation is a viable method for efficient and secure medical data sharing, with the potential to facilitate enhanced collaborative research and clinical applications.", "awesome_sections": [ "Applications / Medical" ], "notes": "kind: evaluation-benchmark rather than application — the paper benchmarks existing DD methods on medical imaging rather than proposing one; families left empty since no single mechanism is central." }, { "id": "att-2024", "type": "paper", "title": "Dataset Distillation by Automatic Training Trajectories", "short": "ATT", "arxiv": "2407.14245", "url": null, "date": "2024-07-19", "venue": { "name": "ECCV 2024", "status": "conference" }, "authors": [ "Dai Liu", "Jindong Gu", "Hu Cao", "Carsten Trinitis", "Martin Schulz" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "trajectory-matching" ], "tier": "catalogued", "links": { "code": "https://github.com/NiaLiu/ATT", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Replaces MTT's fixed synthetic-step count with an automatically and adaptively adjusted trajectory length to counter the 'Accumulated Mismatching Problem' caused by forcing the synthetic set to conform to one fixed-length segment of every expert trajectory, improving cross-architecture generalization and stability over fixed-length trajectory matching.", "abstract": "Dataset Distillation is used to create a concise, yet informative, synthetic dataset that can replace the original dataset for training purposes. Some leading methods in this domain prioritize long-range matching, involving the unrolling of training trajectories with a fixed number of steps (NS) on the synthetic dataset to align with various expert training trajectories. However, traditional long-range matching methods possess an overfitting-like problem, the fixed step size NS forces synthetic dataset to distortedly conform seen expert training trajectories, resulting in a loss of generality-especially to those from unencountered architecture. We refer to this as the Accumulated Mismatching Problem (AMP), and propose a new approach, Automatic Training Trajectories (ATT), which dynamically and adaptively adjusts trajectory length NS to address the AMP. Our method outperforms existing methods particularly in tests involving cross-architectures. Moreover, owing to its adaptive nature, it exhibits enhanced stability in the face of parameter variations.", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ], "builds_on": [ "mtt-2022" ] }, { "id": "ddfad-2024", "type": "paper", "title": "DDFAD: Dataset Distillation Framework for Audio Data", "short": "DDFAD", "arxiv": "2407.10446", "url": null, "date": "2024-07-15", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Wenbo Jiang", "Rui Zhang", "Hongwei Li", "Xiaoyuan Liu", "Haomiao Yang", "Shui Yu" ], "kind": "method", "settings": [ "other-data" ], "families": [ "trajectory-matching" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Applies dataset distillation to audio for the first time: extracts a Fused Differential MFCC feature (FD-MFCC) from each clip, distills it with the matching-training-trajectory method, and reconstructs playable audio from the distilled features with a Griffin-Lim-based algorithm. Reports effectiveness across several audio datasets.", "abstract": "Deep neural networks (DNNs) have achieved significant success in numerous applications. The remarkable performance of DNNs is largely attributed to the availability of massive, high-quality training datasets. However, processing such massive training data requires huge computational and storage resources. Dataset distillation is a promising solution to this problem, offering the capability to compress a large dataset into a smaller distilled dataset. The model trained on the distilled dataset can achieve comparable performance to the model trained on the whole dataset. While dataset distillation has been demonstrated in image data, none have explored dataset distillation for audio data. In this work, for the first time, we propose a Dataset Distillation Framework for Audio Data (DDFAD). Specifically, we first propose the Fused Differential MFCC (FD-MFCC) as extracted features for audio data. After that, the FD-MFCC is distilled through the matching training trajectory distillation method. Finally, we propose an audio signal reconstruction algorithm based on the Griffin-Lim Algorithm to reconstruct the audio signal from the distilled FD-MFCC. Extensive experiments demonstrate the effectiveness of DDFAD on various audio datasets. In addition, we show that DDFAD has promising application prospects in many applications, such as continual learning and neural architecture search.", "notes": "" }, { "id": "fyi-2024", "type": "paper", "title": "FYI: Flip Your Images for Dataset Distillation", "short": "FYI", "arxiv": "2407.08113", "url": null, "date": "2024-07-11", "venue": { "name": "ECCV 2024", "status": "conference" }, "authors": [ "Byunggwan Son", "Youngmin Oh", "Donghyeon Baek", "Bumsub Ham" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/cvlab-yonsei/FYI", "project": "https://cvlab.yonsei.ac.kr/projects/FYI/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Identifies bilateral equivalence -- object parts appearing on one side of a real image tend to recur, mirrored, on other images in the dataset -- which pushes synthesis to duplicate discriminative parts on both sides of synthetic images and limits fine-grained discrimination; FYI embeds horizontal flipping into the distillation process to counter this without changing training objectives or architectures, and integrates into several state-of-the-art methods for consistent gains on CIFAR-10/100, Tiny-ImageNet and ImageNet.", "abstract": "Dataset distillation synthesizes a small set of images from a large-scale real dataset such that synthetic and real images share similar behavioral properties (e.g, distributions of gradients or features) during a training process. Through extensive analyses on current methods and real datasets, together with empirical observations, we provide in this paper two important things to share for dataset distillation. First, object parts that appear on one side of a real image are highly likely to appear on the opposite side of another image within a dataset, which we call the bilateral equivalence. Second, the bilateral equivalence enforces synthetic images to duplicate discriminative parts of objects on both the left and right sides of the images, limiting the recognition of subtle differences between objects. To address this problem, we introduce a surprisingly simple yet effective technique for dataset distillation, dubbed FYI, that enables distilling rich semantics of real images into synthetic ones. To this end, FYI embeds a horizontal flipping technique into distillation processes, mitigating the influence of the bilateral equivalence, while capturing more details of objects. Experiments on CIFAR-10/100, Tiny-ImageNet, and ImageNet demonstrate that FYI can be seamlessly integrated into several state-of-the-art methods, without modifying training objectives and network architectures, and it improves the performance remarkably.", "awesome_sections": [ "Main / Distilled Dataset Parametrization" ], "notes": "Reclassified from parameterization to optimization-recipes: the contribution is an augmentation/training trick during synthesis, not a change to what is stored." }, { "id": "dataset-quantization-with-active-2024", "type": "paper", "title": "Dataset Quantization with Active Learning based Adaptive Sampling", "arxiv": "2407.07268", "url": null, "date": "2024-07-09", "venue": { "name": "ECCV 2024", "status": "conference" }, "authors": [ "Zhenghao Zhao", "Yuzhang Shang", "Junyi Wu", "Yan Yan" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "dataset-quantization" ], "tier": "catalogued", "links": { "code": "https://github.com/ichbill/DQAS", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Shows that uniform per-class sample counts are unnecessary for dataset quantization -- some classes tolerate large reductions with little performance loss -- and proposes an active-learning-based adaptive sampling strategy (DQAS) that reallocates samples from stable to sensitive classes, plus a revised pipeline that uses final-stage quantization features to build more precise bins; outperforms prior dataset compression methods across multiple datasets.", "abstract": "Deep learning has made remarkable progress recently, largely due to the availability of large, well-labeled datasets. However, the training on such datasets elevates costs and computational demands. To address this, various techniques like coreset selection, dataset distillation, and dataset quantization have been explored in the literature. Unlike traditional techniques that depend on uniform sample distributions across different classes, our research demonstrates that maintaining performance is feasible even with uneven distributions. We find that for certain classes, the variation in sample quantity has a minimal impact on performance. Inspired by this observation, an intuitive idea is to reduce the number of samples for stable classes and increase the number of samples for sensitive classes to achieve a better performance with the same sampling ratio. Then the question arises: how can we adaptively select samples from a dataset to achieve optimal performance? In this paper, we propose a novel active learning based adaptive sampling strategy, Dataset Quantization with Active Learning based Adaptive Sampling (DQAS), to optimize the sample selection. In addition, we introduce a novel pipeline for dataset quantization, utilizing feature space from the final stage of dataset quantization to generate more precise dataset bins. Our comprehensive evaluations on the multiple datasets show that our approach outperforms the state-of-the-art dataset compression methods.", "awesome_sections": [ "Main / Dataset Quantization" ] }, { "id": "d3s-2024", "type": "paper", "title": "Large Scale Dataset Distillation with Domain Shift", "short": "D3S", "arxiv": null, "url": "https://openreview.net/forum?id=0FWPKHMCSc", "date": "2024-07", "venue": { "name": "ICML 2024", "status": "conference" }, "authors": [ "Noel Loo", "Alaa Maalouf", "Ramin Hasani", "Mathias Lechner", "Alexander Amini", "Daniela Rus" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "tier": "catalogued", "links": { "code": "https://github.com/yolky/d3s_distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "pmlr", "depth": "abstract" }, "summary": "Reframes large-scale dataset distillation as a domain-shift problem between the synthetic and real data distributions, derives a universal upper bound on the distillation loss under this framing and optimizes it efficiently, reporting state-of-the-art results and improved cross-architecture generalization on Tiny-ImageNet, ImageNet-1K and ImageNet-21K.", "abstract": null, "awesome_sections": [ "Main / Better Optimization" ], "notes": "Non-arXiv; family assignment (distribution-matching) is a best fit for the domain-shift framing, not stated verbatim in the abstract — flagged for review." }, { "id": "gc-bench-2024", "type": "paper", "title": "GC-Bench: An Open and Unified Benchmark for Graph Condensation", "short": "GC-Bench", "arxiv": "2407.00615", "url": null, "date": "2024-06-30", "venue": { "name": "NeurIPS 2024", "status": "conference" }, "authors": [ "Qingyun Sun", "Ziying Chen", "Beining Yang", "Cheng Ji", "Xingcheng Fu", "Sheng Zhou", "Hao Peng", "Jianxin Li", "Philip S. Yu" ], "kind": "evaluation-benchmark", "settings": [ "graph" ], "families": [], "builds_on": [ "gcond-2022", "doscond-2022", "sfgc-2023", "geom-2024", "mirage-2024", "dm-2023" ], "tier": "notable", "links": { "code": "https://github.com/RingBDStack/GC-Bench", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Introduces GC-Bench, a benchmark that systematically evaluates 12 graph-condensation methods on node- and graph-level tasks across 12 datasets along effectiveness, transferability and complexity, and releases a reusable library for training and evaluating graph condensation methods.", "abstract": "Graph condensation (GC) has recently garnered considerable attention due to its ability to reduce large-scale graph datasets while preserving their essential properties. The core concept of GC is to create a smaller, more manageable graph that retains the characteristics of the original graph. Despite the proliferation of graph condensation methods developed in recent years, there is no comprehensive evaluation and in-depth analysis, which creates a great obstacle to understanding the progress in this field. To fill this gap, we develop a comprehensive Graph Condensation Benchmark (GC-Bench) to analyze the performance of graph condensation in different scenarios systematically. Specifically, GC-Bench systematically investigates the characteristics of graph condensation in terms of the following dimensions: effectiveness, transferability, and complexity. We comprehensively evaluate 12 state-of-the-art graph condensation algorithms in node-level and graph-level tasks and analyze their performance in 12 diverse graph datasets. Further, we have developed an easy-to-use library for training and evaluating different GC methods to facilitate reproducible research. The GC-Bench library is available at https://github.com/RingBDStack/GC-Bench.", "awesome_sections": [ "Applications / Graph Neural Network / Benchmark" ], "notes": "", "explained": { "before": "Individual graph condensation papers (GCond, DosCond, SGDD, GCDM/DM, SFGC, GEOM, KiDD, Mirage) each report results under their own protocol and condensation-ratio choices, and a concurrent benchmark (GCondenser) covers only node-level homogeneous-graph methods with limited evaluation dimensions, leaving no unified, cross-family comparison of effectiveness, transferability and efficiency.", "problem": "Reported progress cannot be trusted at face value because methods differ in condensation-ratio ranges, backbones, and evaluated datasets, and several dimensions relevant to real deployment — behavior under a much broader ratio sweep, transfer to non-node-classification tasks, transfer across backbone architectures (including graph transformers), sensitivity to initialization, and actual memory/time cost — are not reported by the original papers at all.", "idea": "Build one open library and evaluation harness that runs 12 representative methods across 6 mechanism categories (coreset, gradient matching, trajectory matching, distribution matching, kernel ridge regression, computation-tree compression) on 12 datasets (5 homogeneous + 2 heterogeneous node-level, 5 graph-level) and both node- and graph-level tasks, under a shared protocol, and use it to answer six specific research questions about effectiveness, structure-sensitivity, task transfer, architecture transfer, initialization, and efficiency rather than only headline accuracy.", "method": "No new condensation objective is proposed. GC-Bench standardizes datasets/splits, backbones and evaluation protocol, then re-runs each method (using authors' released code/hyperparameters where possible) across a wider range of condensation ratios than the original papers used, records node- and graph-classification accuracy, and additionally measures: transfer of a node-classification-condensed graph to link prediction / node clustering / anomaly detection; transfer across 5 GNN backbones plus MLP and Graph Transformer when condensation used a different backbone; sensitivity to 5 initialization schemes (random noise, random sample, center, k-center, k-means); and wall-clock condensation time plus peak CPU/GPU memory to reach best validation performance.", "evidence": "Table 2 (node classification, various GCNs/backbones, hard/method-native labels, broadened ratio sweep): GC methods reach lossless or super-original accuracy in 5/7 datasets at some ratio (e.g. Citeseer r=0.18%, ACM r=0.003%, DBLP r=0.002%), and distribution-matching methods (GCDM, DM) underperform gradient- and trajectory-matching methods on 6/7 datasets. Critically, at the largest ratios tested (e.g. ogbn-arxiv r=5.00%, Flickr r=5.00%, DBLP r=0.186%, Reddit r=5.00%), GCDM, DM, DosCond, GCond and SGDD all hit out-of-memory (OOM) errors, while SFGC and (mostly) GEOM survive — directly contradicting the impression from individual papers that these methods scale gracefully to larger budgets; GEOM itself OOMs on Reddit r=5.00% and r=1.00% despite reporting a \"lossless\" ogbn-arxiv r=5% result in its own paper. Table 3 (graph classification, GIN): KiDD (kernel ridge regression) wins 18/25 cases but depends entirely on structure rather than the backbone, so its results collapse when the downstream model is switched from GIN to GCN — the paper's key cross-model ablation for that method. RQ3 (Fig. 2): all condensed datasets lose substantial accuracy when transferred from node classification to link prediction, clustering, or anomaly detection, with gradient-matching methods (GCond, DosCond, SGDD) transferring better than the structure-free trajectory-matching methods (SFGC, GEOM). RQ4 (Fig. 3): graphs condensed with SGC as backbone transfer well across architectures, but graphs condensed with Graph Transformer transfer poorly in both directions; increasing GCond's gradient-matching steps reduces cross-architecture transfer, implying more matching steps encode more backbone-specific bias. RQ5: initialization choice (e.g. Center vs. Random Noise on Cora) changes convergence speed by up to 71% but has little effect on final accuracy.", "limitations": "Evaluates only node classification and graph classification as condensation objectives (explicitly named as the study's own scope limit); Mirage cannot directly hit the paper's target condensation ratios and had to be matched to DosCond's disk usage instead; some KRR/CTC configurations return no result due to search recursion limits (\"-\" entries); and, like any benchmark, its conclusions are only as current as the 12 methods and hyperparameter settings it reproduced (e.g. no diffusion- or generative-prior graph condensation method is included)." } }, { "id": "mct-2025", "type": "paper", "title": "Towards Stable and Storage-efficient Dataset Distillation: Matching Convexified Trajectory", "short": "MCT", "arxiv": "2406.19827", "url": null, "date": "2024-06-28", "venue": { "name": "CVPR 2025", "status": "conference" }, "authors": [ "Wenliang Zhong", "Haoyu Tang", "Qinghai Zheng", "Mingzhu Xu", "Yupeng Hu", "Liqiang Nie" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "trajectory-matching" ], "tier": "catalogued", "links": { "code": "https://github.com/Zhong0x29a/MCT", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Diagnoses three problems with matching training trajectories (MTT) — instability of SGD-generated expert trajectories, slow convergence, and high storage cost for stored trajectories — and replaces the target trajectory with a convex combination of expert trajectories derived from linearized neural-tangent-kernel dynamics, giving a smaller, continuously samplable, more stable target for the student to match.", "abstract": "The rapid evolution of deep learning and large language models has led to an exponential growth in the demand for training data, prompting the development of Dataset Distillation methods to address the challenges of managing large datasets. Among these, Matching Training Trajectories (MTT) has been a prominent approach, which replicates the training trajectory of an expert network on real data with a synthetic dataset. However, our investigation found that this method suffers from three significant limitations: 1. Instability of expert trajectory generated by Stochastic Gradient Descent (SGD); 2. Low convergence speed of the distillation process; 3. High storage consumption of the expert trajectory. To address these issues, we offer a new perspective on understanding the essence of Dataset Distillation and MTT through a simple transformation of the objective function, and introduce a novel method called Matching Convexified Trajectory (MCT), which aims to provide better guidance for the student trajectory. MCT leverages insights from the linearized dynamics of Neural Tangent Kernel methods to create a convex combination of expert trajectories, guiding the student network to converge rapidly and stably. This trajectory is not only easier to store, but also enables a continuous sampling strategy during distillation, ensuring thorough learning and fitting of the entire expert trajectory. Comprehensive experiments across three public datasets validate the superiority of MCT over traditional MTT methods.", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ], "builds_on": [ "mtt-2022" ] }, { "id": "gc4nc-2024", "type": "paper", "title": "GC4NC: A Benchmark Framework for Graph Condensation on Node Classification with New Insights", "short": "GC4NC", "arxiv": "2406.16715", "url": null, "date": "2024-06-24", "venue": { "name": "NeurIPS 2025 Datasets and Benchmarks Track", "status": "conference" }, "authors": [ "Shengbo Gong", "Juntong Ni", "Noveen Sachdeva", "Carl Yang", "Wei Jin" ], "kind": "evaluation-benchmark", "settings": [ "graph" ], "families": [], "tier": "catalogued", "links": { "code": "https://github.com/Emory-Melody/GraphSlim", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Introduces GC4NC, a benchmark that evaluates graph-condensation methods for node classification across performance, efficiency, privacy preservation, denoising ability, NAS effectiveness and transferability, reporting new insights into which design choices drive condensed-graph quality along these dimensions.", "abstract": "Graph condensation (GC) is an emerging technique designed to learn a significantly smaller graph that retains the essential information of the original graph. This condensed graph has shown promise in accelerating graph neural networks while preserving performance comparable to those achieved with the original, larger graphs. Additionally, this technique facilitates downstream applications like neural architecture search and deepens our understanding of redundancies in large graphs. Despite the rapid development of GC methods, particularly for node classification, a unified evaluation framework is still lacking to systematically compare different GC methods or clarify key design choices for improving their effectiveness. To bridge these gaps, we introduce \\textbf{GC4NC}, a comprehensive framework for evaluating diverse GC methods on node classification across multiple dimensions including performance, efficiency, privacy preservation, denoising ability, NAS effectiveness, and transferability. Our systematic evaluation offers novel insights into how condensed graphs behave and the critical design choices that drive their success. These findings pave the way for future advancements in GC methods, enhancing both performance and expanding their real-world applications. Our code is available at https://github.com/Emory-Melody/GraphSlim/tree/main/benchmark.", "awesome_sections": [ "Applications / Graph Neural Network / Benchmark" ], "notes": "" }, { "id": "behaviour-distillation-2024", "type": "paper", "title": "Behaviour Distillation", "arxiv": "2406.15042", "url": null, "date": "2024-06-21", "venue": { "name": "ICLR 2024", "status": "conference" }, "authors": [ "Andrei Lupu", "Chris Lu", "Jarek Liesen", "Robert Tjarko Lange", "Jakob Foerster" ], "kind": "application", "settings": [ "other-data" ], "families": [ "meta-learning-bptt" ], "applications": [ "rl" ], "tier": "catalogued", "links": { "code": "https://github.com/flairox/behaviour-distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Formalizes behaviour distillation: discovering and condensing the information needed to train an expert policy into a synthetic set of state-action pairs without access to expert data. Introduces HaDES, which searches such datasets with evolution strategies rather than backpropagation, and reports that as few as four state-action pairs can train continuous-control agents to competitive performance, generalizing out of distribution to other architectures and hyperparameters, and to zero-shot multi-task training.", "abstract": "Dataset distillation aims to condense large datasets into a small number of synthetic examples that can be used as drop-in replacements when training new models. It has applications to interpretability, neural architecture search, privacy, and continual learning. Despite strong successes in supervised domains, such methods have not yet been extended to reinforcement learning, where the lack of a fixed dataset renders most distillation methods unusable. Filling the gap, we formalize behaviour distillation, a setting that aims to discover and then condense the information required for training an expert policy into a synthetic dataset of state-action pairs, without access to expert data. We then introduce Hallucinating Datasets with Evolution Strategies (HaDES), a method for behaviour distillation that can discover datasets of just four state-action pairs which, under supervised learning, train agents to competitive performance levels in continuous control tasks. We show that these datasets generalize out of distribution to training policies with a wide range of architectures and hyperparameters. We also demonstrate application to a downstream task, namely training multi-task agents in a zero-shot fashion. Beyond behaviour distillation, HaDES provides significant improvements in neuroevolution for RL over previous approaches and achieves SoTA results on one standard supervised dataset distillation task. Finally, we show that visualizing the synthetic datasets can provide human-interpretable task insights.", "awesome_sections": [ "Applications / Reinforcement Learning" ], "notes": "Family tagged as meta-learning-bptt for the bi-level performance-matching objective, though HaDES optimizes it with evolution strategies rather than backpropagation through time; flagged as a classification doubt." }, { "id": "image-distillation-for-safe-2024", "type": "paper", "title": "Image Distillation for Safe Data Sharing in Histopathology", "short": "InfoDist", "arxiv": "2406.13536", "url": null, "date": "2024-06-19", "venue": { "name": "MICCAI 2024", "status": "conference" }, "authors": [ "Zhe Li", "Bernhard Kainz" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "prior": [ "diffusion" ] }, "tier": "catalogued", "applications": [ "medical" ], "links": { "code": "https://github.com/ZheLi2020/InfoDist", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Trains a latent diffusion model on histopathology data and builds a distilled, human-readable synthetic dataset by selecting the most informative generated images through graph community analysis of the representation space, addressing prior distillation methods' unreadable outputs and weak downstream performance. Reports downstream classifiers trained on the synthetic set reaching performance suitable for practical application, comparable to models trained on real data.", "abstract": "Histopathology can help clinicians make accurate diagnoses, determine disease prognosis, and plan appropriate treatment strategies. As deep learning techniques prove successful in the medical domain, the primary challenges become limited data availability and concerns about data sharing and privacy. Federated learning has addressed this challenge by training models locally and updating parameters on a server. However, issues, such as domain shift and bias, persist and impact overall performance. Dataset distillation presents an alternative approach to overcoming these challenges. It involves creating a small synthetic dataset that encapsulates essential information, which can be shared without constraints. At present, this paradigm is not practicable as current distillation approaches only generate non human readable representations and exhibit insufficient performance for downstream learning tasks. We train a latent diffusion model and construct a new distilled synthetic dataset with a small number of human readable synthetic images. Selection of maximally informative synthetic images is done via graph community analysis of the representation space. We compare downstream classification models trained on our synthetic distillation data to models trained on real data and reach performances suitable for practical application.", "awesome_sections": [ "Applications / Medical" ], "notes": "" }, { "id": "label-worth-2024", "type": "paper", "title": "A Label is Worth a Thousand Images in Dataset Distillation", "short": "A label is worth a thousand images", "arxiv": "2406.10485", "url": null, "date": "2024-06-15", "venue": { "name": "NeurIPS 2024", "status": "conference" }, "authors": [ "Tian Qin", "Zhiwei Deng", "David Alvarez-Melis" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [ "label-distillation" ], "tier": "landmark", "links": { "code": "https://github.com/sunnytqin/no-distillation", "project": null }, "facets": { "labels": "soft-static", "regime": "decoupled", "scale": [ "imagenet-1k", "tiny-imagenet" ] }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "By 2024 nearly every method that scaled to ImageNet-1K relied on soft labels from a pretrained teacher without studying why: SRe2L (2023) decouples synthesis from labeling and assigns 300 teacher-generated soft labels per synthetic image (one per augmented crop); Ra-BPTT and FRePo learn soft labels jointly with images but only scale to IPC 1-2 on downsized ImageNet. Method papers reported gains and attributed them to their image-synthesis technique.", "problem": "Dataset-distillation methods are structurally very different (bi-level meta-learning, trajectory matching, distribution matching) yet nearly all winning ImageNet-scale methods share one ingredient — soft labels — that had not been isolated as a cause. It was unknown how much of the reported accuracy comes from the synthetic images versus from the labels.", "idea": "Freeze the images (use randomly sampled real images, not synthesized ones) and vary only the labels; if a \"soft label baseline\" built from random real images plus teacher soft labels matches state-of-the-art distilled-image methods, the images are not doing the work the field credits them with. Soft labels help only insofar as they carry structured, semantically meaningful information (e.g. inter-class similarity), not because they are merely non-one-hot.", "method": "Optimizes nothing new: the \"soft label baseline\" randomly samples real images per class and labels them with the softmax output of a pretrained expert (checkpoint chosen per IPC budget by early-stopping the expert), replicating SRe2L's per-crop CutMix-augmented relabeling scheme. In the loop: a pretrained classifier (ResNet-18/50 on ImageNet, ConvNet on smaller sets) used only as a label generator, no bi-level optimization. Separately, labels are learned directly via MTT's and truncated-BPTT's objectives with images frozen, to test whether distillation objectives recover the same information as expert soft labels. Structured information is isolated with an \"i-th label swapping test\" (replacing the i-th largest softmax entry with the smallest) and top-k truncation of the softmax vector.", "evidence": "ImageNet-1K, hard labels vs random-image + soft-label baseline vs SRe2L, ResNet-18/50 eval (Table 1): hard-label random images reach 0.6% (IPC1)/20.4% (IPC50); the soft-label baseline with ResNet18-generated labels reaches 6.6%/47.9% and with CutMix-augmented soft labels 2.9%/46.8%, matching or beating SRe2L's reported 2.9%/46.8%-level numbers at equal IPC while storing far less (SRe2L's 300 labels/image cost \"at least 300x\" the soft-label baseline's single label per image). On TinyImageNet/CIFAR-100/CIFAR-10 (Table 2) the soft-label baseline trails Ra-BPTT/MTT/DM at IPC1 but matches or beats them by IPC50-100. Data-knowledge scaling law (Fig. 6): recovering full teacher knowledge needs ~10 IPC with full label information (k=200) but ~40 IPC when the soft label is truncated to the top-32 entries, giving a fitted power law relating soft-label information content to an effective 6x reduction in required data. Zero-shot ablation (Fig. 7): removing all images of a class barely hurts accuracy on that class if its soft-label information is kept elsewhere, but zeroing out a class's entries in the labels (keeping the images) collapses accuracy on that class to 0% at IPC1 — labels carry more of the \"content\" than images. Label swapping test (Fig. 4) shows swapping top-ranked softmax entries hurts far more than swapping low-ranked ones, confirming the useful information concentrates in a few structured entries. BPTT-learned labels (images frozen) converge to the same labels as an early-stopped expert ensemble (Fig. 8, minimal Jensen-Shannon distance at a consistent epoch), while MTT fails to learn useful labels this way.", "limitations": "The paper explicitly restricts to fixing images and only varying labels, so it cannot rule out that jointly learning images and labels does better than either alone; it explicitly notes recent methods still beat the soft-label baseline at low IPC on smaller datasets and concludes both images and labels matter. It only tests image classification (no other modality). The soft-label baseline itself still requires a pretrained expert and per-crop relabeling, so it does not resolve the label-storage cost it implicitly documents (300x for SRe2L-style regenerated labels). No wall-clock/GPU-hour cost comparison is reported." }, "summary": "Ablates the many disparate dataset-distillation methods and finds that the single largest common factor in their reported performance is the use of soft (probabilistic) labels rather than their specific synthesis techniques; shows the soft labels must carry structured information to help, provides empirical scaling laws relating soft-label effectiveness to images-per-class, and establishes an empirical Pareto frontier for data-efficient learning, challenging the attribution of gains to image-synthesis method alone.", "abstract": "Data $\\textit{quality}$ is a crucial factor in the performance of machine learning models, a principle that dataset distillation methods exploit by compressing training datasets into much smaller counterparts that maintain similar downstream performance. Understanding how and why data distillation methods work is vital not only for improving these methods but also for revealing fundamental characteristics of \"good\" training data. However, a major challenge in achieving this goal is the observation that distillation approaches, which rely on sophisticated but mostly disparate methods to generate synthetic data, have little in common with each other. In this work, we highlight a largely overlooked aspect common to most of these methods: the use of soft (probabilistic) labels. Through a series of ablation experiments, we study the role of soft labels in depth. Our results reveal that the main factor explaining the performance of state-of-the-art distillation methods is not the specific techniques used to generate synthetic data but rather the use of soft labels. Furthermore, we demonstrate that not all soft labels are created equal; they must contain $\\textit{structured information}$ to be beneficial. We also provide empirical scaling laws that characterize the effectiveness of soft labels as a function of images-per-class in the distilled dataset and establish an empirical Pareto frontier for data-efficient learning. Combined, our findings challenge conventional wisdom in dataset distillation, underscore the importance of soft labels in learning, and suggest new directions for improving distillation methods. Code for all experiments is available at https://github.com/sunnytqin/no-distillation.", "awesome_sections": [ "Main / Label Distillation" ] }, { "id": "h-glad-2025", "type": "paper", "title": "Hierarchical Features Matter: A Deep Exploration of Progressive Parameterization Method for Dataset Distillation", "short": "H-GLaD", "arxiv": "2406.05704", "url": null, "date": "2024-06-09", "venue": { "name": "CVPR 2025", "status": "conference" }, "authors": [ "Xinhao Zhong", "Hao Fang", "Bin Chen", "Xulin Gu", "Meikang Qiu", "Shuhan Qi", "Shu-Tao Xia" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "generative-latent-optimization", "parameterization" ], "facets": { "parameterization": [ "generative-latent" ], "labels": "hard", "prior": [ "gan" ], "regime": "single-level", "scale": [ "mnist-cifar", "imagenet-subsets", "imagenet-1k" ] }, "tier": "notable", "links": { "code": "https://github.com/ndhg1213/H-GLaD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Extends GLaD-style generative-prior distillation by systematically exploring hierarchical feature layers within a pretrained GAN, rather than a single fixed latent space, since different layers offer different informative guidance for distillation; also introduces a class-relevant feature-distance metric that lowers the cost of evaluating synthetic sets. Matches or surpasses diffusion-based generative distillation at extreme compression (IPC=1, IPC=10) at comparable time cost.", "abstract": "Dataset distillation is an emerging dataset reduction method, which condenses large-scale datasets while maintaining task accuracy. Current parameterization methods achieve enhanced performance under extremely high compression ratio by optimizing determined synthetic dataset in informative feature domain. However, they limit themselves to a fixed optimization space for distillation, neglecting the diverse guidance across different informative latent spaces. To overcome this limitation, we propose a novel parameterization method dubbed Hierarchical Parameterization Distillation (H-PD), to systematically explore hierarchical feature within provided feature space (e.g., layers within pre-trained generative adversarial networks). We verify the correctness of our insights by applying the hierarchical optimization strategy on GAN-based parameterization method. In addition, we introduce a novel class-relevant feature distance metric to alleviate the computational burden associated with synthetic dataset evaluation, bridging the gap between synthetic and original datasets. Experimental results demonstrate that the proposed H-PD achieves a significant performance improvement under various settings with equivalent time consumption, and even surpasses current generative distillation using diffusion models under extreme compression ratios IPC=1 and IPC=10.", "builds_on": [ "glad-2023", "dc-2021", "dsa-2021", "dm-2023", "mtt-2022", "tesla-2023", "sre2l-2023", "it-gan-2022" ], "awesome_sections": [ "Main / Generative Distillation / GAN" ], "explained": { "before": "GLaD fixes one intermediate StyleGAN-XL layer, chosen by manual preliminary search, as the optimization space for a given matching method (DC/DSA/DM/MTT/TESLA); the paper shows this fixed choice is architecture- and dataset-dependent, and can even underperform plain pixel-space distillation in the same-architecture setting it was tuned for (Fig. 1, e.g. TESLA+GLaD 50.7% vs. Pixel 51.7% on ImageNet-A, Table 1).", "problem": "A single fixed GAN layer cannot serve every combination of dataset, matching method and IPC well, and finding the right layer by manual, per-setting search is itself expensive; meanwhile diffusion-based generative distillation (Minimax, D4M) degrades toward coreset-like random sampling at the extreme compression ratios (IPC=1/10) that GAN-latent optimization methods target.", "idea": "Do not commit to one GAN layer: progressively optimize through every hierarchical layer of the frozen generator in sequence, carrying the best latent from one layer forward as the next layer's initialization, and replace expensive real-accuracy evaluation of intermediate candidates with a cheap class-relevant feature-distance proxy (gradient-weighted class-activation-map distance) so the layer search stays affordable.", "method": "The same frozen, per-dataset StyleGAN-XL used by GLaD is decomposed into layers $G_{K-1}\\circ\\cdots\\circ G_0$. Starting from an averaged-noise latent (mean of several Gaussian samples through the mapping network, avoiding costly GAN-inversion/clustering initialization), each layer $G_i$ is optimized for a fixed number of SGD steps (100 for TESLA/DSA, 20 for DM) against the chosen matching loss $\\mathcal{L}= \\mathcal{M}(\\phi(G_{K-1}\\circ\\cdots\\circ G_i(z_i)),\\phi(\\mathcal{T}))$; within that budget the latent with lowest class-relevant feature distance $\\mathcal{D}(\\mathcal{S},\\mathcal{T})$ (Eq. 6, built from a pretrained feature extractor's activations weighted by the CAM gradient of the true class) is kept as $z_i^*$ and passed through $G_i$ to initialize the next layer. After traversing all layers to the pixel domain, the single best-performing intermediate synthetic set (by real validation accuracy, tracked only across the small number of per-layer candidates) is output. The generator itself is never fine-tuned; only the latents are optimized, exactly as in GLaD. The framework is validated on top of DC, DSA, DM, MTT/TESLA (bilevel/matching methods) and, separately, on top of SRe2L (decoupled, BN-statistics-matching synthesis) to reach Tiny-ImageNet and full ImageNet-1K.", "evidence": "ImageNet-Subset (128x128) · IPC=1 · same architecture as backbone · hard labels (Table 1): H-PD beats both Pixel and GLaD for every base method, e.g. TESLA: Pixel 51.7% -> GLaD 50.7% (worse than pixel) -> H-PD 55.1% on ImageNet-A; DM: Pixel 39.4% -> GLaD 41.0% -> H-PD 42.8%. Cross-architecture (Table 5, IPC=1): TESLA ImageNet-A Pixel 33.4% -> GLaD 39.9% -> H-PD 40.2%; DM ImageNet-A Pixel 27.2% -> GLaD 31.6% -> H-PD 34.9%. Against diffusion-based generative methods at IPC=1 on ImageNette/ImageWoof/Birds/Fruits/Cats, re-run under GLaD's hard-label, no-mixup, no-training-time-matching protocol for a fair comparison (Table 6): H-PD 45.4%/28.3%/39.7%/25.6%/29.6% vs. Minimax 22.8%/17.8%/23.2%/17.5%/19.8% vs. D4M 15.2%/17.4%/18.2%/17.6%/23.4%, i.e. optimization-based GAN-latent search greatly outperforms diffusion sampling once both are stripped to the same hard-label, no-teacher, extreme-IPC protocol. Extended to decoupled synthesis via SRe2L on Tiny-ImageNet and ImageNet-1K, ResNet-18 (Table 2): at IPC-1, Pixel 0.1% -> GLaD 1.2% -> H-PD 2.6% on ImageNet-1K; at IPC-10, 21.3% -> 21.9% -> 23.5%. Cost (Table 7, ImageNet-A-E average): H-PD roughly matches or beats GLaD's wall-clock time despite searching more layers, e.g. DM: GLaD 64 min / 37.4% vs. H-PD 15 min / 39.1% (fewer optimization steps per layer for DM). Key ablation (Table 8, incremental): starting from GLaD-TESLA 51.9% (ImageNet-B), adding averaged-noise initialization alone gives 53.5%, adding hierarchical-layer search on top gives 56.2%, and adding the class-relevant distance metric on top of that gives 57.4% — hierarchical search is the dominant component, and the distance-metric gain is only accessible once the space is unfixed.", "limitations": "Only validated as an add-on to GAN-latent parameterization (StyleGAN-XL), not tested with diffusion generators; still requires a StyleGAN-XL trained on (or matched to) the target dataset, so it inherits GLaD's generator-availability dependency and does not test out-of-distribution/random generators as GLaD did; the per-layer greedy search with early stopping by a proxy metric is not guaranteed to find the jointly optimal layer/latent combination; absolute accuracy at full ImageNet-1K IPC=1 remains very low (2.6%) even though it triples GLaD's number; the paper notes the released GLaD code lacks TESLA's CIFAR-10 augmentation/hyperparameters, which depresses the ConvNet baseline it compares against on CIFAR-10 (Table 4)." } }, { "id": "what-is-dd-learning-2024", "type": "paper", "title": "What is Dataset Distillation Learning?", "short": "What is DD learning?", "arxiv": "2406.04284", "url": null, "date": "2024-06-06", "venue": { "name": "ICML 2024", "status": "conference" }, "authors": [ "William Yang", "Ye Zhu", "Zhiwei Deng", "Olga Russakovsky" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [], "facets": { "scale": [ "mnist-cifar" ] }, "tier": "notable", "links": { "code": "https://github.com/princetonvisualai/What-is-Dataset-Distillation-Learning", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "By mid-2024 the field had four families of small-scale synthesis objectives — BPTT (bi-level meta-model matching), distribution matching (DM), gradient matching (DC/GM), and trajectory matching (MTT) — each shown to reach high accuracy on CIFAR-10, but no prior work examined what information the resulting synthetic images actually encode once distillation is done. Zhong & Liu (2023, cited) had already shown distilled data does not transfer well across architectures.", "problem": "High evaluation accuracy does not explain *how* a handful of synthetic images substitute for a full training set: is it representing the data manifold, mimicking a trained model's parameters, or something else? Without knowing what is stored, method design and evaluation are both guesswork.", "idea": "Distilled data behaves like a compressed record of a real model's *early training dynamics* rather than a compressed sample of the data manifold: a model trained on distilled data ends up similar to a real model that was heavily regularized (early-stopped), not to a model trained on a small random subset of real data, and individual distilled points still carry distinct, humanly interpretable semantic content (e.g. \"yellow car\", \"military plane in the sky\") extractable via influence functions.", "method": "Uses one representative method per paradigm — BPTT (meta-model/bi-level), DM (distribution matching, random conv nets), DC/gradient matching, and MTT (trajectory matching, expert trajectories in the loop) — on CIFAR-10 with a 3-layer ConvNet trained via SGD for 300 iterations. No new synthesis method is proposed. Analysis tools: (1) an inverse pipeline — train on real data, evaluate on distilled images — to test whether distilled data lies on the real manifold and is recognizable; (2) prediction-agreement comparison between distilled-trained models and early-stopped vs. data-subset-trained models; (3) Hessian-trace loss-curvature analysis (Hutchinson's method / PyHessian) of a real-data-trained model evaluated against distilled images over training iterations; (4) exact leave-one-out influence functions $I_{x_d\\to x_t}=L(x_t;\\hat\\theta_{-x_d})-L(x_t;\\hat\\theta)$ computed by retraining, plus LLaVA-generated semantic attribute annotations of real images to interpret which real-image concepts each distilled image is most influential for.", "evidence": "CIFAR-10, ConvNet, all four synthesis methods (BPTT, DM, GM, MTT) at matched accuracy (no absolute accuracy is the point; the comparisons are all controlled at fixed accuracy or fixed data budget): real-trained models classify distilled images with high accuracy (Fig. 2) and distilled images sit inside real class clusters under UMAP, showing recognizability; yet distilled pixel values fall outside [0,1] and clipping BPTT images to [0,1] drops retrained accuracy from 58% to 44%, showing distilled data is off the real manifold. Mixing 10 distilled images/class with 0-250 real images/class (Fig. 3 right) *decreases* accuracy relative to distilled-only training, in sharp contrast to a random-real-image baseline that improves monotonically with more data — direct evidence that a reported distilled-set accuracy is not simply \"more of the same kind of value\" as real images and cannot be naively combined with them. Prediction-agreement analysis (Fig. 4, at matched test accuracy): distilled-trained models agree far more with early-stopped real models (iteration 35-130 depending on method) than with models trained on random 0.5-5% real subsets. A real-data model's classification accuracy on distilled images plateaus after ~150 of 300 training iterations even as its accuracy on the real test set keeps rising (Fig. 5), and loss-curvature (Hessian trace) with respect to BPTT/MTT-distilled images flattens within ~1.5 epochs (Fig. 6), both indicating distilled data encodes only early-training information. Influence functions show weak correlation with feature-space visual similarity (Fig. 7, confirming influence is not merely \"looks similar\"), are consistent across random seeds (Fig. 8 left) and independent of which other distilled images are present (Fig. 8 right, correlation preserved when leave-one-out uses real images instead of other distilled images). Precision-recall analysis against LLaVA-derived semantic tags (Fig. 9, Table 1) finds individual BPTT/MTT distilled images are predictive of specific non-class semantics (e.g. \"yellow car\", \"parking lot\") beyond the one-hot label.", "limitations": "All experiments are on CIFAR-10 with small ConvNets and small-scale methods (BPTT, DM, GM, MTT); the paper does not test decoupled/relabel+KD methods (SRe2L-style) or ImageNet scale, so it is unclear whether \"compresses early-training dynamics\" still describes methods that rely on a pretrained teacher rather than a bi-level/surrogate objective. The curvature analysis explicitly could not draw the same clean conclusion for distribution/gradient matching data (curvature fluctuates rather than flattening cleanly), so the \"early training dynamics\" story is best supported for BPTT/MTT specifically. Influence functions require expensive leave-one-out retraining (only tractable at the 100-image scale studied) and the semantic-extraction step depends on a separate multimodal model (LLaVA) to generate ground-truth attributes, which is itself an added source of noise/bias not present in the class labels." }, "summary": "Finds that distilled data cannot substitute for real data outside the standard dataset-distillation evaluation setting, that the distillation process retains high task performance mainly by compressing information about real models' early training dynamics, and provides an interpretation framework showing individual distilled data points carry meaningful semantic information.", "abstract": "Dataset distillation has emerged as a strategy to overcome the hurdles associated with large datasets by learning a compact set of synthetic data that retains essential information from the original dataset. While distilled data can be used to train high performing models, little is understood about how the information is stored. In this study, we posit and answer three questions about the behavior, representativeness, and point-wise information content of distilled data. We reveal distilled data cannot serve as a substitute for real data during training outside the standard evaluation setting for dataset distillation. Additionally, the distillation process retains high task performance by compressing information related to the early training dynamics of real models. Finally, we provide an framework for interpreting distilled data and reveal that individual distilled data points contain meaningful semantic information. This investigation sheds light on the intricate nature of distilled data, providing a better understanding on how they can be effectively utilized.", "awesome_sections": [ "Main / Better Understanding" ] }, { "id": "lors-2024", "type": "paper", "title": "Low-Rank Similarity Mining for Multimodal Dataset Distillation", "short": "LoRS", "arxiv": "2406.03793", "url": null, "date": "2024-06-06", "venue": { "name": "ICML 2024", "status": "conference" }, "authors": [ "Yue Xu", "Zhilin Lin", "Yusong Qiu", "Cewu Lu", "Yong-Lu Li" ], "kind": "method", "settings": [ "vision-language" ], "families": [ "trajectory-matching", "label-distillation" ], "facets": { "labels": "similarity-matrix", "prior": [ "trained-experts" ], "regime": "bilevel", "parameterization": [ "pixels", "embedding-features", "similarity-matrix" ], "scale": [ "flickr30k-coco" ] }, "tier": "core", "links": { "code": "https://github.com/silicx/LoRS_Distill", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "LoRS distills a ground-truth image-text similarity matrix alongside the synthetic pairs, instead of assuming one caption strictly matches one image, and stores that similarity structure efficiently via low-rank factorization. The paper reports significant improvements over prior vision-language distillation algorithms and proposes LoRS as a foundational synthetic-data setup for image-text dataset distillation.", "abstract": "Though dataset distillation has witnessed rapid development in recent years, the distillation of multimodal data, e.g., image-text pairs, poses unique and under-explored challenges. Unlike unimodal data, image-text contrastive learning (ITC) data lack inherent categorization and should instead place greater emphasis on modality correspondence. In this work, we propose Low-Rank Similarity Mining (LoRS) for multimodal dataset distillation, that concurrently distills a ground truth similarity matrix with image-text pairs, and leverages low-rank factorization for efficiency and scalability. The proposed approach brings significant improvement to the existing algorithms, marking a significant contribution to the field of visual-language dataset distillation. We advocate adopting LoRS as a foundational synthetic data setup for image-text dataset distillation. Our code is available at https://github.com/silicx/LoRS_Distill.", "builds_on": [ "mtt-vl-2024" ], "explained": { "before": "MTT-VL was the first vision-language distillation method, but it (implicitly) treats each synthetic image as matching exactly one synthetic caption one-to-one, the same assumption single-modal trajectory matching makes about image-label pairs.", "problem": "Real image-text corpora are not one-to-one: a caption can describe many images and an image can match several captions, and image-text contrastive (ITC) training depends on the whole similarity structure between a batch's images and texts, not just matched pairs. Distilling only paired samples throws away that structure.", "idea": "Distill a full image-text similarity matrix alongside the synthetic pairs, so the synthetic set carries its own soft ground-truth affinities rather than an implicit identity matrix; make this affordable by factorizing the matrix into a diagonal plus a low-rank term instead of storing it densely.", "method": "Synthetic image pixels $\\tilde{\\mathcal X}$, synthetic text embeddings $\\tilde{\\mathcal Y}$, and a learnable similarity matrix $\\tilde S=\\omega I+\\frac{\\alpha}{r}LR^\\top$ (diagonal $\\omega$ plus rank-$r$ factors $L,R\\in\\mathbb{R}^{N\\times r}$) are jointly optimized. An NFNet image encoder (pretrained, trained during distillation) and a BERT-base text encoder (pretrained, frozen for efficiency) plus a trainable projection produce embeddings. The outer objective is standard multi-step trajectory matching (MTT): $\\|\\tilde\\theta_t-\\theta_T^*\\|^2/\\|\\theta_0^*-\\theta_T^*\\|^2$ on the image tower; the inner student loss on the synthetic batch is a weighted binary cross-entropy (wBCE) over $\\tilde S$ rather than the usual InfoNCE with an implicit identity target, so the student is trained to reproduce the learned soft affinities, not just match its own pairs.", "evidence": "With NFNet+BERT on Flickr30K, LoRS$_{\\text{wBCE}}$ reports IR@1/5/10 = 8.3/24.1/35.1 and TR@1/5/10 = 11.8/35.8/49.2 at 100 pairs, rising to IR@1/5/10 = 10.0/28.9/41.6 and TR@1/5/10 = 15.5/39.8/53.7 at 500 pairs (Table 2). On COCO at 100/500 pairs, IR@1 = 1.8/2.8 and TR@1 = 3.3/5.3 (Table 3). Baselines (coreset methods, MTT-VL, and an author-adapted TESLA$_{\\text{wBCE}}$) are re-run by the authors under the same protocol rather than copied. The key ablation (Table 5, 200 pairs) isolates the loss and the low-rank structure: replacing wBCE with plain BCE collapses IR@1 from 8.6 to 0.6; removing the similarity matrix entirely (identity matrix, i.e. classic one-to-one matching) drops performance further to IR@1 = 0.2; removing only the low-rank $L,R$ term (keeping diagonal $\\omega$ only) gives IR@1 = 7.4, still above the identity-matrix ablation but below the full factorization (8.6), showing both the learned similarity structure and its low-rank correction matter. Storage and time overhead are small: with $r=20$-$50$ the similarity matrix adds about 0.07-0.3% extra storage and about 0.8% extra training time (6.35s/iteration vs 6.47s baseline at 100 pairs, Table 10), measured on one RTX 4090.", "limitations": "The paper only distills text as continuous embeddings, not tokens, and states that direct distillation of text tokens is left to future work. It notes an explicit storage/performance trade-off in the rank $r$, and that the wBCE loss was chosen empirically and would need to be redesigned for other contrastive objectives. Absolute recall is still far below full-data training (e.g. IR@1 = 10.0 vs a full-dataset ceiling around 23-27 reported by later papers using the same protocol), and evaluation is limited to the NFNet+frozen-BERT encoder pair used during distillation." }, "notes": "" }, { "id": "dd-bias-2024", "type": "paper", "title": "Mitigating Bias in Dataset Distillation", "short": "Bias in DD", "arxiv": "2406.06609", "url": null, "date": "2024-06-06", "venue": { "name": "ICML 2024", "status": "conference" }, "authors": [ "Justin Cui", "Ruochen Wang", "Yuanhao Xiong", "Cho-Jui Hsieh" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Finds that color and background biases present in the original dataset are amplified by dataset distillation while corruption bias is instead suppressed, and introduces a kernel-density-estimation-based sample reweighting scheme that reduces this amplification, reporting 91.5% accuracy versus 23.8% for vanilla DM on biased CMNIST (5% bias-conflict ratio) at IPC-50.", "abstract": "Dataset Distillation has emerged as a technique for compressing large datasets into smaller synthetic counterparts, facilitating downstream training tasks. In this paper, we study the impact of bias inside the original dataset on the performance of dataset distillation. With a comprehensive empirical evaluation on canonical datasets with color, corruption and background biases, we found that color and background biases in the original dataset will be amplified through the distillation process, resulting in a notable decline in the performance of models trained on the distilled dataset, while corruption bias is suppressed through the distillation process. To reduce bias amplification in dataset distillation, we introduce a simple yet highly effective approach based on a sample reweighting scheme utilizing kernel density estimation. Empirical results on multiple real-world and synthetic datasets demonstrate the effectiveness of the proposed method. Notably, on CMNIST with 5% bias-conflict ratio and IPC 50, our method achieves 91.5% test accuracy compared to 23.8% from vanilla DM, boosting the performance by 67.7%, whereas applying state-of-the-art debiasing method on the same dataset only achieves 53.7% accuracy. Our findings highlight the importance of addressing biases in dataset distillation and provide a promising avenue to address bias amplification in the process.", "awesome_sections": [ "Main / Better Understanding" ], "applications": [ "fairness" ], "builds_on": [ "dm-2023" ] }, { "id": "dataset-distillation-generative-model-2024", "type": "paper", "title": "Dataset-Distillation Generative Model for Speech Emotion Recognition", "arxiv": "2406.02963", "url": null, "date": "2024-06-05", "venue": { "name": "Interspeech 2024", "status": "conference" }, "authors": [ "Fabian Ritter-Gutierrez", "Kuan-Po Huang", "Jeremy H. M Wong", "Dianwen Ng", "Hung-yi Lee", "Nancy F. Chen", "Eng Siong Chng" ], "kind": "method", "settings": [ "other-data" ], "families": [ "generator-as-dataset" ], "facets": { "prior": [ "gan" ], "parameterization": [ "generator-weights" ] }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Introduces the first dataset-distillation approach for speech, applied to Speech Emotion Recognition on IEMOCAP: a GAN is trained not to reconstruct real utterances but to distill discriminative information useful for downstream training, then replaces the dataset as a sampler of any desired synthetic size; reports comparable performance under the original class imbalance and a 0.3-point absolute UAR gain under balanced classes, with about 95% less storage and faster downstream training.", "abstract": "Deep learning models for speech rely on large datasets, presenting computational challenges. Yet, performance hinges on training data size. Dataset Distillation (DD) aims to learn a smaller dataset without much performance degradation when training with it. DD has been investigated in computer vision but not yet in speech. This paper presents the first approach for DD to speech targeting Speech Emotion Recognition on IEMOCAP. We employ Generative Adversarial Networks (GANs) not to mimic real data but to distil key discriminative information of IEMOCAP that is useful for downstream training. The GAN then replaces the original dataset and can sample custom synthetic dataset sizes. It performs comparably when following the original class imbalance but improves performance by 0.3% absolute UAR with balanced classes. It also reduces dataset storage and accelerates downstream training by 95% in both cases and reduces speaker information which could help for a privacy application.", "awesome_sections": [ "Applications / Speech" ], "notes": "" }, { "id": "condtsf-2024", "type": "paper", "title": "CondTSF: One-line Plugin of Dataset Condensation for Time Series Forecasting", "short": "CondTSF", "arxiv": "2406.02131", "url": null, "date": "2024-06-04", "venue": { "name": "NeurIPS 2024", "status": "conference" }, "authors": [ "Jianrong Ding", "Zhanyu Liu", "Guanjie Zheng", "Haiming Jin", "Linghe Kong" ], "kind": "method", "settings": [ "time-series" ], "families": [ "optimization-recipes" ], "builds_on": [ "mtt-2022", "dc-2021", "kip-2021", "frepo-2022" ], "tier": "catalogued", "links": { "code": "https://github.com/RafaDD/CondTSF", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Analyzes why classification-style dataset-condensation objectives are misaligned with time-series forecasting, where evaluation requires closeness of predictions rather than matching discrete labels, and proposes CondTSF, a one-line plugin that can be added to existing time-series condensation methods to shrink the gap between full-data and synthetic-data forecast predictions; reports consistent gains across eight benchmark datasets, largest at low condensation ratios.", "abstract": "Dataset condensation is a newborn technique that generates a small dataset that can be used in training deep neural networks to lower training costs. The objective of dataset condensation is to ensure that the model trained with the synthetic dataset can perform comparably to the model trained with full datasets. However, existing methods predominantly concentrate on classification tasks, posing challenges in their adaptation to time series forecasting (TS-forecasting). This challenge arises from disparities in the evaluation of synthetic data. In classification, the synthetic data is considered well-distilled if the model trained with the full dataset and the model trained with the synthetic dataset yield identical labels for the same input, regardless of variations in output logits distribution. Conversely, in TS-forecasting, the effectiveness of synthetic data distillation is determined by the distance between predictions of the two models. The synthetic data is deemed well-distilled only when all data points within the predictions are similar. Consequently, TS-forecasting has a more rigorous evaluation methodology compared to classification. To mitigate this gap, we theoretically analyze the optimization objective of dataset condensation for TS-forecasting and propose a new one-line plugin of dataset condensation designated as Dataset Condensation for Time Series Forecasting (CondTSF) based on our analysis. Plugging CondTSF into previous dataset condensation methods facilitates a reduction in the distance between the predictions of the model trained with the full dataset and the model trained with the synthetic dataset, thereby enhancing performance. We conduct extensive experiments on eight commonly used time series datasets. CondTSF consistently improves the performance of all previous dataset condensation methods across all datasets, particularly at low condensing ratios.", "awesome_sections": [ "Applications / Time Series" ], "notes": "", "explained": { "before": "Parameter-matching dataset condensation methods (DC's gradient matching, MTT's trajectory matching, and kernel-closed-form methods KIP/FRePo) were designed for classification, where a synthetic set is \"well-distilled\" once a model trained on it assigns the same discrete class label as a model trained on the full data, regardless of logit magnitude; applying them unmodified to time-series forecasting only matches model parameters, not what the paper argues actually matters for forecasting.", "problem": "Forecasting has no discrete label to match — its correctness criterion is the pointwise distance between the two models' continuous predictions, which the paper calls a strictly more rigorous similarity criterion than classification's label-only agreement. Naively imported condensation methods only optimize a term implicitly related to overall parameter closeness and miss a separate term the paper identifies as the dominant contributor at low condensation ratios.", "idea": "Derive (via a first-order Taylor expansion around the synthetic data point) an upper bound on the forecasting test-error objective that splits cleanly into two additive terms — a \"gradient term\" (distance between the two trained models' parameters/gradients, which existing parameter-matching methods already optimize) and a \"value term\" (distance between the two models' predicted values on the synthetic input itself) — and add a lightweight, gradient-free update that directly minimizes the value term, since prior methods leave it unaddressed.", "method": "Theorem 1 bounds $\\mathcal{L}_{test}$ by a constant irreducible label-noise term plus the value term $\\|\\mathcal{M}_{\\theta_{s,test}}(s)-\\mathcal{M}_{\\theta_{f,test}}(s)\\|^2$ plus the gradient term $\\|(\\nabla\\mathcal{M}_{\\theta_{s,test}}(s)-\\nabla\\mathcal{M}_{\\theta_{f,test}}(s))^\\top(x-s)\\|^2$, derived assuming (or first-order-approximating) a linear forecaster so gradient = model parameters. The gradient term is what existing trajectory/gradient-matching methods already minimize (proven via Cauchy-Schwarz, reducing to $\\|\\theta_s-\\theta_f\\|^2$). For the value term, Theorem 2 shows $\\theta_ {f,test}$'s prediction on synthetic input $s$ (though $\\theta_{f,test}$ itself is unavailable at condensation time, since it is only sampled at test time) can be approximated by the prediction of any \"expert\" model $\\theta_f^i$ from the offline-trained parameter buffer, because all such experts predict similarly on a given input regardless of their random initialization (empirically confirmed via MDS visualization, Fig. 3). CondTSF is then a one-line, gradient-free additive update applied every $G$ epochs, interleaved with the backbone method's normal matching steps: $s^{(i+1)}_{label} = (1-\\beta) s^{(i)}_{label} + \\beta\\,\\mathcal{M}_{\\theta_f^i}(s^{(i)}_{input})$, which is proven to shrink the label error exponentially at rate $(1-\\beta)^2$ per update. No new network, generator, or label scheme is introduced; CondTSF only touches how the synthetic sequence's target/label values are updated.", "evidence": "Table 1 (ExchangeRate/Weather/Electricity/Traffic, distilled length 48, MAE and MSE, backbones DC and presumably others, 5 synthetic datasets averaged): CondTSF reduces test error for every backbone on every dataset, e.g. DC on ExchangeRate: MSE drops 27.8% (0.875→0.632); smallest gains on DC specifically (1.7-11.7% MSE reduction) versus larger gains reported for direct parameter-matching backbones like MTT, because (per Sec. 5.2) DC only indirectly matches gradients-of-gradients each iteration, so its parameter error accumulates and CondTSF's value-term fix has less room to help. Table 3 (cross-architecture, DLinear-distilled synthetic data evaluated by training MLP/LSTM/CNN from scratch): MTT+CondTSF outperforms Random, DC, KIP, FRePo, and plain MTT across ExchangeRate and Weather on both architectures shown. Figure 4 (ExchangeRate, MTT vs. MTT+CondTSF): parameter error (gradient term) converges similarly with or without CondTSF, confirming CondTSF does not interfere with gradient-term optimization, while label error (value term) rises under plain MTT but stays low under MTT+CondTSF, and test error tracks the value-term difference — direct evidence the value term, not the gradient term, is the mechanism CondTSF fixes. Per the paper's own summary, gains are \"particularly\" large at low condensation ratios (distill ratios as low as 0.83‰-6.33‰ across the eight datasets, Table 2).", "limitations": "The theoretical derivation explicitly assumes a linear forecaster (so that gradient = model parameter, letting the two terms decompose cleanly); the paper states this analysis \"no longer holds\" for more complex nonlinear models, and while an appendix reports CondTSF still helps with non-linear experts and non-parameter-matching backbones, that generalization is empirical, not covered by the theorem; CondTSF consistently helps least on DC among the backbones tested, and the paper attributes this to a mechanism (indirect, per-iteration gradient matching accumulating parameter error) it does not fully resolve; evaluation uses fairly short forecasting windows (24-in, 24-out) and small synthetic-set sizes (length 48, one training pair per synthetic set)." } }, { "id": "dance-2024", "type": "paper", "title": "DANCE: Dual-View Distribution Alignment for Dataset Condensation", "short": "DANCE", "arxiv": "2406.01063", "url": null, "date": "2024-06-03", "venue": { "name": "IJCAI 2024", "status": "conference" }, "authors": [ "Hansong Zhang", "Shikun Li", "Fanzhao Lin", "Weiping Wang", "Zhenxing Qian", "Shiming Ge" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "tier": "catalogued", "links": { "code": "https://github.com/Hansong-Zhang/DANCE", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "parameterization": [ "pixels" ], "labels": "hard", "prior": [ "random-networks", "pretrained-classifier" ], "regime": "single-level", "scale": [ "mnist-cifar", "tiny-imagenet", "imagenet-subsets" ] }, "explained": { "before": "dm-2023 and its refinements idm-2023 and cafe-2022 rely on random or lightly-trained embeddings that the paper argues fail in two ways: \"Persistent Training\" (a single random/briefly-trained snapshot cannot proxy the long trajectory a real classifier goes through) and \"Distribution Shift\" (unconstrained synthetic features can drift outside the real class region).", "problem": "Fixes like IDM's model queue or CAFE's feature alignment require training multiple models from scratch during condensation -- expensive and hyperparameter-heavy -- and still do not correct for synthetic features leaving their class region, which changes the decision boundary a downstream classifier learns.", "idea": "Use a handful of pretrained, reusable \"expert\" encoders at intermediate stages of training to give distribution matching both a pseudo long-term view of feature evolution (inner-class) and a classification-based calibration that keeps synthetic features inside the correct class region (inter-class), while keeping distribution matching's single-level optimization.", "method": "Expert models are pretrained once, offline (60 epochs for low-resolution datasets and TinyImageNet, 80 epochs for ImageNet-1K subsets), and reused across all IPC settings. The inner-class Pseudo Long-Term Distribution Alignment (PLTDA) loss interpolates between a random encoder and an expert encoder with a random mixing coefficient (lambda ~ U(0,1)) and matches class-wise mean features under this \"middle\" encoder. The inter-class Distribution Calibration loss is a cross-entropy loss from a fully trained expert classifier evaluated on the synthetic images, pulling them back toward the correct decision region. No bi-level loop -- only the synthetic images are updated.", "evidence": "CIFAR-10, IPC=10, ConvNet-3: DANCE 70.8% vs. DM 48.9%. CIFAR-100, IPC=50: 52.8% vs. DM 43.6%. TinyImageNet, IPC=10: 26.4% vs. IDM 21.9%. ImageSquawk (128x128), IPC=10: 77.2% vs. DM 50.4%. Cost: per-iteration 0.12s / 3045MB vs. DM's 0.08s / 3455MB on CIFAR-10 -- about 50% slower per step but slightly lower peak memory, since expert pretraining is amortized offline and reused across IPC settings rather than counted against each run. Ablation (CIFAR-10, IPC=10): PLTDA alone 64.8%, calibration alone 65.6%, both 70.8% -- the inner- and inter-class views are complementary. A single expert already gives 69.2%, with only marginal gains up to 15+ experts.", "limitations": "Requires dataset-specific pretrained expert models before condensation can even start -- an offline but real cost that is conceptually close to trajectory matching's expert-trajectory precomputation, though cheaper since only a few checkpoints (not full trajectories) are needed. The paper does not quantify or directly compare this offline pretraining cost against MTT-style expert-trajectory generation. New datasets or domains require re-pretraining the experts before DANCE can be applied." }, "summary": "Improves distribution matching with an inner-class 'pseudo long-term' alignment through several intermediate pretrained encoders and an inter-class distribution-calibration term from expert models, addressing the persistent-training and distribution-shift weaknesses of plain distribution matching while keeping its efficiency.", "abstract": "Dataset condensation addresses the problem of data burden by learning a small synthetic training set that preserves essential knowledge from the larger real training set. To date, the state-of-the-art (SOTA) results are often yielded by optimization-oriented methods, but their inefficiency hinders their application to realistic datasets. On the other hand, the Distribution-Matching (DM) methods show remarkable efficiency but sub-optimal results compared to optimization-oriented methods. In this paper, we reveal the limitations of current DM-based methods from the inner-class and inter-class views, i.e., Persistent Training and Distribution Shift. To address these problems, we propose a new DM-based method named Dual-view distribution AligNment for dataset CondEnsation (DANCE), which exploits a few pre-trained models to improve DM from both inner-class and inter-class views. Specifically, from the inner-class view, we construct multiple \"middle encoders\" to perform pseudo long-term distribution alignment, making the condensed set a good proxy of the real one during the whole training process; while from the inter-class view, we use the expert models to perform distribution calibration, ensuring the synthetic data remains in the real class region during condensing. Experiments demonstrate the proposed method achieves a SOTA performance while maintaining comparable efficiency with the original DM across various scenarios. Source codes are available at https://github.com/Hansong-Zhang/DANCE.", "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ], "builds_on": [ "dm-2023", "idm-2023", "cafe-2022" ] }, { "id": "bacon-2024", "type": "paper", "title": "BACON: Bayesian Optimal Condensation Framework for Dataset Distillation", "short": "BACON", "arxiv": "2406.01112", "url": null, "date": "2024-06-03", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Zheng Zhou", "Hongbo Zhao", "Guangliang Cheng", "Xiangtai Li", "Shuchang Lyu", "Wenquan Feng", "Qi Zhao" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/zhouzhengqd/BACON", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Casts dataset distillation as minimizing an expected risk functional over the joint data distribution under a Bayesian framework and derives a numerically tractable lower bound as an approximate solution, reporting a 3.46-point accuracy gain over IDM at IPC-10 on CIFAR-10 and 3.10 points on Tiny-ImageNet when combined with existing methods.", "abstract": "Dataset Distillation (DD) aims to distill knowledge from extensive datasets into more compact ones while preserving performance on the test set, thereby reducing storage costs and training expenses. However, existing methods often suffer from computational intensity, particularly exhibiting suboptimal performance with large dataset sizes due to the lack of a robust theoretical framework for analyzing the DD problem. To address these challenges, we propose the BAyesian optimal CONdensation framework (BACON), which is the first work to introduce the Bayesian theoretical framework to the literature of DD. This framework provides theoretical support for enhancing the performance of DD. Furthermore, BACON formulates the DD problem as the minimization of the expected risk function in joint probability distributions using the Bayesian framework. Additionally, by analyzing the expected risk function for optimal condensation, we derive a numerically feasible lower bound based on specific assumptions, providing an approximate solution for BACON. We validate BACON across several datasets, demonstrating its superior performance compared to existing state-of-the-art methods. For instance, under the IPC-10 setting, BACON achieves a 3.46% accuracy gain over the IDM method on the CIFAR-10 dataset and a 3.10% gain on the TinyImageNet dataset. Our extensive experiments confirm the effectiveness of BACON and its seamless integration with existing methods, thereby enhancing their performance for the DD task. Code and distilled datasets are available at BACON.", "awesome_sections": [ "Main / Better Optimization" ], "builds_on": [ "idm-2023" ] }, { "id": "adaptive-backdoor-attacks-against-2024", "type": "paper", "title": "Adaptive Backdoor Attacks Against Dataset Distillation for Federated Learning", "arxiv": null, "url": "https://ieeexplore.ieee.org/document/10622462/", "date": "2024-06-01", "venue": { "name": "ICC 2024", "status": "conference" }, "authors": [ "Ze Chai", "Zhipeng Gao", "Yijing Lin", "Chen Zhao", "Xinlei Yu", "Zhiqiang Xie" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [], "applications": [ "federated-learning", "backdoor" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "ieee", "depth": "abstract" }, "summary": "Proposes three adaptive dataset-condensation-based backdoor attacks against dataset distillation used in federated learning, in which the trigger is dynamically modified during the distillation process rather than fixed in advance.", "abstract": null, "awesome_sections": [ "Applications / Privacy" ], "notes": "Verified via IEEE Xplore listing and a search summary of the paper (full text behind paywall); authors and page range (4614-4619) taken from the search result, not read directly." }, { "id": "lqm-2024", "type": "paper", "title": "Dataset Condensation with Latent Quantile Matching", "short": "LQM", "arxiv": null, "url": "https://openaccess.thecvf.com/content/CVPR2024W/DDCV/html/Wei_Dataset_Condensation_with_Latent_Quantile_Matching_CVPRW_2024_paper.html", "date": "2024-06", "venue": { "name": "CVPR 2024 Workshop", "status": "workshop" }, "authors": [ "Wei Wei", "Tom De Schepper", "Kevin Mets" ], "kind": "method", "settings": [ "image-classification", "graph" ], "families": [ "distribution-matching" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "cvf", "depth": "abstract" }, "summary": "Shows that matching only the mean of latent feature embeddings, as in standard distribution matching, lets very different distributions appear identical, and proposes Latent Quantile Matching, which instead matches the quantiles of the embedding distributions to minimize a goodness-of-fit statistic; matches or beats prior distribution-matching methods on image and graph-structured datasets and improves continual graph learning.", "abstract": null, "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ], "builds_on": [ "dm-2023" ] }, { "id": "selmatch-2024", "type": "paper", "title": "SelMatch: Effectively Scaling Up Dataset Distillation via Selection-Based Initialization and Partial Updates by Trajectory Matching", "short": "SelMatch", "arxiv": "2406.18561", "url": null, "date": "2024-05-28", "venue": { "name": "ICML 2024", "status": "conference" }, "authors": [ "Yongmin Lee", "Hye Won Chung" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "trajectory-matching", "coreset-selection" ], "tier": "notable", "links": { "code": "https://github.com/Yongalls/SelMatch", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Finds that trajectory-matching distillation fails to close the coverage gap between easy and hard test samples as images-per-class grows, and addresses this by initializing the synthetic set with selection-based (coreset) samples and only partially updating them through trajectory matching, tuning the selection-to-update ratio to the target IPC; consistently outperforms leading selection-only and distillation-only methods on CIFAR-10/100 and Tiny-ImageNet from 5% to 30% subset ratios.", "abstract": "Dataset distillation aims to synthesize a small number of images per class (IPC) from a large dataset to approximate full dataset training with minimal performance loss. While effective in very small IPC ranges, many distillation methods become less effective, even underperforming random sample selection, as IPC increases. Our examination of state-of-the-art trajectory-matching based distillation methods across various IPC scales reveals that these methods struggle to incorporate the complex, rare features of harder samples into the synthetic dataset even with the increased IPC, resulting in a persistent coverage gap between easy and hard test samples. Motivated by such observations, we introduce SelMatch, a novel distillation method that effectively scales with IPC. SelMatch uses selection-based initialization and partial updates through trajectory matching to manage the synthetic dataset's desired difficulty level tailored to IPC scales. When tested on CIFAR-10/100 and TinyImageNet, SelMatch consistently outperforms leading selection-only and distillation-only methods across subset ratios from 5% to 30%.", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ], "builds_on": [ "mtt-2022" ] }, { "id": "calibrated-dataset-condensation-for-2024", "type": "paper", "title": "Calibrated Dataset Condensation for Faster Hyperparameter Search", "short": "HCDC", "arxiv": "2405.17535", "url": null, "date": "2024-05-27", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Mucong Ding", "Yuancheng Xu", "Tahseen Rabbani", "Xiaoyu Liu", "Brian Gravelle", "Teresa Ranadive", "Tai-Ching Tuan", "Furong Huang" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "gradient-matching" ], "applications": [ "nas" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Rather than matching gradients to preserve accuracy, HCDC condenses a synthetic validation set so that the ranking of models by hyperparameter/architecture choice is preserved between the condensed and real validation sets, matching hyperparameter gradients via implicit differentiation and an approximate inverse Hessian. The paper reports the condensed validation set keeps model rankings stable and speeds up hyperparameter and architecture search on both image and graph tasks.", "abstract": "Dataset condensation can be used to reduce the computational cost of training multiple models on a large dataset by condensing the training dataset into a small synthetic set. State-of-the-art approaches rely on matching the model gradients between the real and synthetic data. However, there is no theoretical guarantee of the generalizability of the condensed data: data condensation often generalizes poorly across hyperparameters/architectures in practice. This paper considers a different condensation objective specifically geared toward hyperparameter search. We aim to generate a synthetic validation dataset so that the validation-performance rankings of the models, with different hyperparameters, on the condensed and original datasets are comparable. We propose a novel hyperparameter-calibrated dataset condensation (HCDC) algorithm, which obtains the synthetic validation dataset by matching the hyperparameter gradients computed via implicit differentiation and efficient inverse Hessian approximation. Experiments demonstrate that the proposed framework effectively maintains the validation-performance rankings of models and speeds up hyperparameter/architecture search for tasks on both images and graphs.", "awesome_sections": [ "Applications / Neural Architecture Search" ], "notes": "" }, { "id": "rela-2024", "type": "paper", "title": "Efficiency for Free: Ideal Data Are Transportable Representations", "short": "ReLA", "arxiv": "2405.14669", "url": null, "date": "2024-05-23", "venue": { "name": "NeurIPS 2024", "status": "conference" }, "authors": [ "Peng Sun", "Yi Jiang", "Tao Lin" ], "kind": "method", "settings": [ "self-supervised" ], "families": [ "label-distillation" ], "facets": { "labels": "learned" }, "tier": "notable", "links": { "code": "https://github.com/LINs-lab/ReLA", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "ReLA (Representation Learning Accelerator) argues that a publicly available, task- and architecture-agnostic \"prior model\" can be used to produce efficient training signal that is transportable across tasks and architectures, substituting learned representation targets for the labels/pipeline a representation-learning method would otherwise need. The paper reports that using a CIFAR-10-pretrained ResNet-18 as the prior model to inform ResNet-50 training on ImageNet-1K cuts computational cost by 50% while matching the accuracy of the original BYOL recipe trained at full cost.", "abstract": "Data, the seminal opportunity and challenge in modern machine learning, currently constrains the scalability of representation learning and impedes the pace of model evolution. In this work, we investigate the efficiency properties of data from both optimization and generalization perspectives. Our theoretical and empirical analysis reveals an unexpected finding: for a given task, utilizing a publicly available, task- and architecture-agnostic model (referred to as the `prior model' in this paper) can effectively produce efficient data. Building on this insight, we propose the Representation Learning Accelerator (\\algopt), which promotes the formation and utilization of efficient data, thereby accelerating representation learning. Utilizing a ResNet-18 pre-trained on CIFAR-10 as a prior model to inform ResNet-50 training on ImageNet-1K reduces computational costs by 50% while maintaining the same accuracy as the model trained with the original BYOL, which requires 100% cost. Our code is available at: \\url{https://github.com/LINs-lab/ReLA}.", "notes": "Mechanism is unusual for this atlas's four families (it accelerates representation learning via a prior model's targets rather than synthesizing a smaller image set); classified under label-distillation as the closest fit, with real doubt about whether it belongs to any P1-P4 family at all (see handoff doubts)." }, { "id": "gift-2025", "type": "paper", "title": "GIFT: Unlocking Full Potential of Labels in Distilled Dataset at Near-zero Cost", "short": "GIFT", "arxiv": "2405.14736", "url": null, "date": "2024-05-23", "venue": { "name": "ICLR 2025", "status": "conference" }, "authors": [ "Xinyi Shang", "Peng Sun", "Tao Lin" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "label-distillation" ], "facets": { "labels": "soft-relabel", "regime": "decoupled", "scale": [ "imagenet-1k", "tiny-imagenet", "mnist-cifar" ] }, "tier": "notable", "links": { "code": "https://github.com/LINs-lab/GIFT", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "Decoupled/teacher-relabel methods (SRe2L, RDED, G-VBSM, CDA) and trajectory-matching methods with learned soft labels (DATM) each pick their own loss for training the student on soft labels (e.g. SRe2L uses KL divergence) without justifying the choice or testing sensitivity to it; RDED evaluates specifically with the AdamW optimizer.", "problem": "Different distillation methods use different, unexamined loss functions for soft-label supervision, and the paper shows this choice moves accuracy by double digits: swapping SRe2L's KL-divergence loss for soft cross-entropy costs 13.3 points on Tiny-ImageNet (IPC10), and changing RDED's optimizer from AdamW to Adam (keeping its KL loss) drops accuracy from 47.5% to 17.8% (Table 8). So headline numbers partly reflect a loss/optimizer pairing tuned to one recipe, not the intrinsic value of the synthetic images or labels.", "idea": "Two changes make soft-label training both stronger and robust to optimizer choice at no added compute: (1) refine the stored soft label by blending it with a smoothed hard label (correcting for the teacher's own errors and adding inter-class dispersion the soft label alone lacks), and (2) replace the training loss with a cosine-similarity-based loss, which a mutual-information argument (an upper bound on InfoNCE/V-information restated via cosine similarity, Theorem 1) shows is close to optimal and which is scale-invariant, removing the loss-magnitude sensitivity that makes Adam/AdamW updates blow up or vanish.", "method": "Plug-and-play, applied at evaluation/training time only (no change to how images or the original labels were synthesized): (1) Label refinement combines the pre-generated soft label y-tilde with a label-smoothed hard label y via a convex combination y_refined = gamma*y/||y|| + (1-gamma)*y-tilde/||y-tilde||, gamma=0.1 found optimal empirically; (2) trains the student with a cosine-similarity loss between student output and refined label instead of KL/cross-entropy. In the loop: whatever teacher(s)/experts the base method (SRe2L/RDED/G-VBSM/CDA/DATM) already used to generate soft labels; GIFT adds no new network and no new forward/backward pass beyond the loss computation.", "evidence": "All results add GIFT on top of an existing method's fixed images+soft-labels, i.e. isolate the label-utilization loss under the teacher relabel-with-KD regime. CIFAR-100/Tiny-ImageNet, ConvNet eval (Table 1): consistent gains across SRe2L/RDED/DATM/G-VBSM, e.g. SRe2L+GIFT +4.3 points on CIFAR-100 IPC10 (33.7->38.0), RDED+GIFT +2.5 (47.5->50.6); on ResNet-18 eval gains are smaller (+0.3 to +1.6). ImageNet-1K (Table 2, ConvNet/ResNet-18): RDED+GIFT +1.8 at IPC10/ResNet-18 (41.4->43.2), G-VBSM+GIFT +1.2; gains shrink as IPC grows (e.g. RDED +0.5 at IPC100). Higher-IPC CDA comparison (Table 3) up to IPC200 on ImageNet-1K: CDA+GIFT +0.5 (63.4->63.9). Cross-optimizer generalization (Table 8, IPC10): the headline effect — RDED with SGD goes from 1.9% to 53.4% (+51.5) with GIFT, and with Adam from 17.8% to 53.7% (+35.9); SRe2L with SGD goes from 1.5% to 43.0% (+42.6) on CIFAR-100 and 0.6% to 43.8% (+43.2) on Tiny-ImageNet — showing the base methods are nearly untrainable with SGD/Adam without GIFT and were implicitly tuned to AdamW. Cost (Table 6, IPC10, ResNet-18): GIFT adds no extra memory and only 1-70 seconds of extra training time out of ~2000s total on ImageNet-1K (~3% overhead), confirming the \"near-zero cost\" claim. Ablation (Section 5.6) shows both label refinement and the cosine loss contribute; the cosine loss is compared against alternative loss functions and found best, and gamma=0.1 is the tuned optimum (Figure 2).", "limitations": "All reported gains are relative to a fixed base method's images and initial soft labels — GIFT cannot fix a poor synthesis method, only how its labels are used, so it never appears as a standalone comparison against random-image or coreset baselines under its own protocol. Gains shrink as IPC grows and are smaller on ResNet-18 than ConvNet, suggesting benefits partly compensate for weaknesses in low-data/loss-mismatch regimes rather than adding new information. The paper does not report a no-GIFT-no-teacher (hard label) baseline in its main tables, so it cannot be used to gauge how much of overall accuracy soft labels contribute versus this specific loss-function fix. DATM's ImageNet-1K synthetic data was unavailable so DATM+GIFT is untested at that scale. No formal \"limitations\" section is given by the authors." }, "summary": "Shows that models trained on distilled data are highly sensitive to the loss function used for soft labels, and introduces GIFT, a plug-and-play soft-label refinement plus cosine-similarity loss that extracts more of the label information already present; reports consistent gains across distillation methods and scales, including a 30.8% cross-optimizer generalization improvement over RDED on ImageNet-1K at IPC=10, at no extra compute cost.", "abstract": "Recent advancements in dataset distillation have demonstrated the significant benefits of employing soft labels generated by pre-trained teacher models. In this paper, we introduce a novel perspective by emphasizing the full utilization of labels. We first conduct a comprehensive comparison of various loss functions for soft label utilization in dataset distillation, revealing that the model trained on the synthetic dataset exhibits high sensitivity to the choice of loss function for soft label utilization. This finding highlights the necessity of a universal loss function for training models on synthetic datasets. Building on these insights, we introduce an extremely simple yet surprisingly effective plug-and-play approach, GIFT, which encompasses soft label refinement and a cosine similarity-based loss function to efficiently leverage full label information. Extensive experiments indicate that GIFT consistently enhances state-of-the-art dataset distillation methods across various dataset scales, without incurring additional computational costs. Importantly, GIFT significantly enhances cross-optimizer generalization, an area previously overlooked. For instance, on ImageNet-1K with IPC = 10, GIFT enhances the state-of-the-art method RDED by 30.8% in cross-optimizer generalization. Our code is available at https://github.com/LINs-lab/GIFT.", "builds_on": [ "rded-2024" ], "awesome_sections": [ "Main / Label Distillation" ] }, { "id": "gcondenser-2024", "type": "paper", "title": "GCondenser: Benchmarking Graph Condensation", "short": "GCondenser", "arxiv": "2405.14246", "url": null, "date": "2024-05-23", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Yilun Liu", "Ruihong Qiu", "Zi Huang" ], "kind": "evaluation-benchmark", "settings": [ "graph" ], "families": [], "tier": "catalogued", "links": { "code": "https://github.com/superallen13/GCondenser", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Introduces GCondenser, the first large-scale graph-condensation benchmark with a standardized condensation-validation-evaluation pipeline that extends to new methods and datasets, and uses it to comprehensively compare mainstream graph-condensation methods.", "abstract": "Large-scale graphs are valuable for graph representation learning, yet the abundant data in these graphs hinders the efficiency of the training process. Graph condensation (GC) alleviates this issue by compressing the large graph into a significantly smaller one that still supports effective model training. Although recent research has introduced various approaches to improve the effectiveness of the condensed graph, comprehensive and practical evaluations across different GC methods are neglected. This paper proposes the first large-scale graph condensation benchmark, GCondenser, to holistically evaluate and compare mainstream GC methods. GCondenser includes a standardised GC paradigm, consisting of condensation, validation, and evaluation procedures, as well as enabling extensions to new GC methods and datasets. With GCondenser, a comprehensive performance study is conducted, presenting the effectiveness of existing methods. GCondenser is open-sourced and available at https://github.com/superallen13/GCondenser.", "awesome_sections": [ "Applications / Graph Neural Network / Benchmark" ], "notes": "" }, { "id": "overcoming-data-and-model-2024", "type": "paper", "title": "Overcoming Data and Model Heterogeneities in Decentralized Federated Learning via Synthetic Anchors", "short": "DeSA", "arxiv": "2405.11525", "url": null, "date": "2024-05-19", "venue": { "name": "ICML 2024", "status": "conference" }, "authors": [ "Chun-Yin Huang", "Kartik Srinivas", "Xin Zhang", "Xiaoxiao Li" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "applications": [ "federated-learning" ], "tier": "notable", "links": { "code": "https://github.com/ubc-tea/DESA", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "In serverless decentralized federated learning, DeSA synthesizes global anchor data from each client's raw data distribution and uses it for two local regularizers: aligning each client's latent embeddings to the anchors, and a knowledge-distillation loss that lets clients learn from one another, grounded in domain-adaptation and KD theory. Reports improved inter- and intra-domain accuracy across clients with diverse data and model heterogeneity.", "abstract": "Conventional Federated Learning (FL) involves collaborative training of a global model while maintaining user data privacy. One of its branches, decentralized FL, is a serverless network that allows clients to own and optimize different local models separately, which results in saving management and communication resources. Despite the promising advancements in decentralized FL, it may reduce model generalizability due to lacking a global model. In this scenario, managing data and model heterogeneity among clients becomes a crucial problem, which poses a unique challenge that must be overcome: How can every client's local model learn generalizable representation in a decentralized manner? To address this challenge, we propose a novel Decentralized FL technique by introducing Synthetic Anchors, dubbed as DeSA. Based on the theory of domain adaptation and Knowledge Distillation (KD), we theoretically and empirically show that synthesizing global anchors based on raw data distribution facilitates mutual knowledge transfer. We further design two effective regularization terms for local training: 1) REG loss that regularizes the distribution of the client's latent embedding with the anchors and 2) KD loss that enables clients to learn from others. Through extensive experiments on diverse client data distributions, we showcase the effectiveness of DeSA in enhancing both inter- and intra-domain accuracy of each client.", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "" }, { "id": "curriculum-dataset-distillation-2025", "type": "paper", "title": "Curriculum Dataset Distillation", "arxiv": "2405.09150", "url": null, "date": "2024-05-15", "venue": { "name": "TIP 2025", "status": "journal" }, "authors": [ "Zhiheng Ma", "Anjia Cao", "Funing Yang", "Yihong Gong", "Xing Wei" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "tier": "catalogued", "links": { "code": "https://github.com/MIV-XJTU/CUDD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Synthesizes decoupled ImageNet-scale images along a simple-to-complex curriculum with curriculum evaluation, plus an adversarial term against the distilling network, to counter the homogeneity and simplicity of prior batch-to-global synthesis; reports gains of 11.1% on Tiny-ImageNet, 9.0% on ImageNet-1K and 7.3% on ImageNet-21K over prior large-scale distillation, with improved cross-architecture generalization and robustness to noise.", "abstract": "Most dataset distillation methods struggle to accommodate large-scale datasets due to their substantial computational and memory requirements. Recent research has begun to explore scalable disentanglement methods. However, there are still performance bottlenecks and room for optimization in this direction. In this paper, we present a curriculum-based dataset distillation framework aiming to harmonize performance and scalability. This framework strategically distills synthetic images, adhering to a curriculum that transitions from simple to complex. By incorporating curriculum evaluation, we address the issue of previous methods generating images that tend to be homogeneous and simplistic, doing so at a manageable computational cost. Furthermore, we introduce adversarial optimization towards synthetic images to further improve their representativeness and safeguard against their overfitting to the neural network involved in distilling. This enhances the generalization capability of the distilled images across various neural network architectures and also increases their robustness to noise. Extensive experiments demonstrate that our framework sets new benchmarks in large-scale dataset distillation, achieving substantial improvements of 11.1\\% on Tiny-ImageNet, 9.0\\% on ImageNet-1K, and 7.3\\% on ImageNet-21K. Our distilled datasets and code are available at https://github.com/MIV-XJTU/CUDD.", "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "an-aggregation-free-federated-2024", "type": "paper", "title": "An Aggregation-Free Federated Learning for Tackling Data Heterogeneity", "short": "FedAF", "arxiv": "2404.18962", "url": null, "date": "2024-04-29", "venue": { "name": "CVPR 2024", "status": "conference" }, "authors": [ "Yuan Wang", "Huazhu Fu", "Renuga Kanagavelu", "Qingsong Wei", "Yong Liu", "Rick Siow Mong Goh" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "facets": { "labels": "soft-static" }, "applications": [ "federated-learning" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "FedAF replaces iterative model aggregation with a collaborative distribution-matching condensation step, where clients jointly learn condensed data and soft labels from peer knowledge that the server then trains the global model on directly. It reports faster convergence and higher global-model accuracy than prior one-shot and iterative FL baselines under label- and feature-skew heterogeneity.", "abstract": "The performance of Federated Learning (FL) hinges on the effectiveness of utilizing knowledge from distributed datasets. Traditional FL methods adopt an aggregate-then-adapt framework, where clients update local models based on a global model aggregated by the server from the previous training round. This process can cause client drift, especially with significant cross-client data heterogeneity, impacting model performance and convergence of the FL algorithm. To address these challenges, we introduce FedAF, a novel aggregation-free FL algorithm. In this framework, clients collaboratively learn condensed data by leveraging peer knowledge, the server subsequently trains the global model using the condensed data and soft labels received from the clients. FedAF inherently avoids the issue of client drift, enhances the quality of condensed data amid notable data heterogeneity, and improves the global model performance. Extensive numerical studies on several popular benchmark datasets show FedAF surpasses various state-of-the-art FL algorithms in handling label-skew and feature-skew data heterogeneity, leading to superior global model accuracy and faster convergence.", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "" }, { "id": "generative-dataset-distillation-balancing-2024", "type": "paper", "title": "Generative Dataset Distillation: Balancing Global Structure and Local Details", "arxiv": "2404.17732", "url": null, "date": "2024-04-26", "venue": { "name": "CVPR 2024 Workshop", "status": "workshop" }, "authors": [ "Longzhen Li", "Guang Li", "Ren Togo", "Keisuke Maeda", "Takahiro Ogawa", "Miki Haseyama" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "generator-as-dataset" ], "facets": { "parameterization": [ "generator-weights" ], "regime": "generator-fine-tuning" }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Trains a conditional GAN to serve as the distilled artifact while explicitly balancing global structural attributes against local details such as texture and shape, which prior generative distillation methods weighted toward high-level semantics alone; continuously optimizes the generator toward a more information-dense synthetic dataset.", "abstract": "In this paper, we propose a new dataset distillation method that considers balancing global structure and local details when distilling the information from a large dataset into a generative model. Dataset distillation has been proposed to reduce the size of the required dataset when training models. The conventional dataset distillation methods face the problem of long redeployment time and poor cross-architecture performance. Moreover, previous methods focused too much on the high-level semantic attributes between the synthetic dataset and the original dataset while ignoring the local features such as texture and shape. Based on the above understanding, we propose a new method for distilling the original image dataset into a generative model. Our method involves using a conditional generative adversarial network to generate the distilled dataset. Subsequently, we ensure balancing global structure and local details in the distillation process, continuously optimizing the generator for more information-dense dataset generation.", "awesome_sections": [ "Main / Generative Distillation / GAN" ], "notes": "Reclassified from generative-latent-optimization to generator-as-dataset: the method trains a conditional GAN as the stored artifact rather than optimizing latents of a frozen generator." }, { "id": "distilled-datamodel-with-reverse-2024", "type": "paper", "title": "Distilled Datamodel with Reverse Gradient Matching", "arxiv": "2404.14006", "url": null, "date": "2024-04-22", "venue": { "name": "CVPR 2024", "status": "conference" }, "authors": [ "Jingwen Ye", "Ruonan Yu", "Songhua Liu", "Xinchao Wang" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "gradient-matching" ], "applications": [ "data-attribution" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Approximates each training sample's influence on a pretrained model without leave-one-out retraining, by first distilling the training set into a synset via reverse gradient matching (offline) and then using that synset to speed up leave-one-out evaluation and compute an attribution matrix (online). Reports training-data-attribution and data-quality evaluations comparable to direct retraining at substantially lower cost.", "abstract": "The proliferation of large-scale AI models trained on extensive datasets has revolutionized machine learning. With these models taking on increasingly central roles in various applications, the need to understand their behavior and enhance interpretability has become paramount. To investigate the impact of changes in training data on a pre-trained model, a common approach is leave-one-out retraining. This entails systematically altering the training dataset by removing specific samples to observe resulting changes within the model. However, retraining the model for each altered dataset presents a significant computational challenge, given the need to perform this operation for every dataset variation. In this paper, we introduce an efficient framework for assessing data impact, comprising offline training and online evaluation stages. During the offline training phase, we approximate the influence of training data on the target model through a distilled synset, formulated as a reversed gradient matching problem. For online evaluation, we expedite the leave-one-out process using the synset, which is then utilized to compute the attribution matrix based on the evaluation objective. Experimental evaluations, including training data attribution and assessments of data quality, demonstrate that our proposed method achieves comparable model behavior evaluation while significantly speeding up the process compared to the direct retraining method.", "awesome_sections": [ "Applications / Machine Unlearning" ], "notes": "" }, { "id": "edc-2024", "type": "paper", "title": "Elucidating the Design Space of Dataset Condensation", "short": "EDC", "arxiv": "2404.13733", "url": null, "date": "2024-04-21", "venue": { "name": "NeurIPS 2024", "status": "conference" }, "authors": [ "Shitong Shao", "Zikai Zhou", "Huanran Chen", "Zhiqiang Shen" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "tier": "core", "links": { "code": "https://github.com/shaoshitong/EDC", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "labels": "soft-relabel", "prior": [ "pretrained-classifier" ], "regime": "decoupled", "parameterization": [ "pixels" ], "scale": [ "imagenet-1k", "imagenet-21k", "tiny-imagenet", "mnist-cifar" ] }, "explained": { "before": "Directly re-evaluates SRe2L, G-VBSM and RDED under one unified protocol and shows each left design choices unexplored or unexplained: noise initialization, coarse global-only BN matching, a sharp loss landscape, and — specifically for RDED — an unablated smoothing learning-rate schedule and small evaluation batch size that the paper shows account for 16.2 of RDED's reported points on their own (RDED alone: 25.8% to 42.0% once those tricks are added).", "problem": "Because prior papers each explored only a slice of the possible design space (synthesis initialization, matching granularity, loss-landscape flatness, post-evaluation schedule/batch size), reported gaps between methods conflate genuine methodological advances with unexplained evaluation-recipe differences, making comparisons across SRe2L/G-VBSM/RDED unreliable.", "idea": "Systematically ablate every design axis across the synthesis, soft-label-generation and post-evaluation stages, keep only the components with an empirical or theoretical justification, and assemble them into one recipe (EDC) evaluated identically across methods.", "method": "Teacher ensemble for ImageNet-1K synthesis: ResNet-18, MobileNetV2, ShuffleNetV2, EfficientNetV2/B0, AlexNet (torchvision-pretrained); relabeling swaps EfficientNetV2 for AlexNet. Recovery keeps BN-statistics-plus-logit matching but adds \"soft category-aware matching\": a weighted mix of global statistical matching and per-class (Gaussian-mixture, one component per class) statistical matching, $\\alpha{=}0.5$ found best, justified by a KL-divergence argument that per-class matching alone would otherwise leave a nonzero divergence floor. Synthesis is initialized from a training-free, RDED-style real-crop condensed set rather than noise, argued via optimal transport to lower the transport cost to the real distribution and shown to roughly halve the needed recovery iterations. A smoothing cosine learning-rate schedule with deceleration factor $\\zeta{=}2$ replaces the standard cosine schedule. Flatness regularization uses an EMA teacher copy of the synthetic batch and restricts the KL term to the observer model's logits, cheaper than SAM but matching or beating it. Relabeling follows SRe2L/G-VBSM-style FKD with RandomResizedCrop+RandomHorizontalFlip (RandAugment on smaller sets); no explicit storage size is reported. Post-eval: ImageNet-1K 300 epochs, AdamW, batch 100, EMA-averaged student (rate 0.99); smaller datasets use 1000 epochs.", "evidence": "ImageNet-1K, ResNet-18, IPC=10 (Table 1): EDC 48.6% vs SRe2L 21.3%, G-VBSM 31.4%, RDED 42.0% — the latter three re-run by EDC under its own protocol and matching their own papers' numbers closely (RDED matches its own 42.0% exactly; SRe2L/G-VBSM match their own IPC=10 figures). IPC=50: EDC 58.0% vs SRe2L 46.8%, G-VBSM 51.8%, RDED 56.5%. Compression ratio 0.78% = 10,000 synthetic images / ~1.28M real training images at IPC=10. CIFAR-100 IPC=10: EDC 63.7% vs RDED 42.6%, G-VBSM 59.5%. Tiny- ImageNet IPC=50: EDC 57.2% vs RDED 58.2% — RDED edges out EDC here, an exception the paper does not resolve. Synthesis cost (Table 23, 4xRTX4090): roughly halves synthesis time (about 9.8h down to about 5h) while raising accuracy, mainly from the real-image-initialization reduction in iterations. Key ablation (Table 3): real-image init + smoothing LR + small batch alone raise a RDED-style baseline from 34.4% to 38.7% (ResNet-18) at zero extra cost; Table 5 isolates soft category-aware matching + weak augmentation + EMA-eval together adding roughly +2.4 points on top of that. No hard-label ablation or result is reported.", "limitations": "States the design space explored, while broad, may still miss valuable strategies, and that EDC's advantage over RDED shrinks as IPC grows (it loses to RDED at Tiny-ImageNet IPC=50). Applying EDC to a new dataset or task requires substantial re-tuning of its hyperparameters or even redesigning the statistical matching, an open challenge the paper does not solve. Still fully dependent on multiple pretrained teacher/observer networks for both synthesis and relabeling, with no reduction in that dependency relative to SRe2L, G-VBSM or RDED." }, "builds_on": [ "sre2l-2023", "g-vbsm-2024", "rded-2024" ], "summary": "Systematically explores the design space of decoupled dataset condensation across scales, contributing soft category-aware matching and a tuned learning-rate schedule grounded in empirical and theoretical analysis; establishes a benchmark for small- and large-scale condensation, reporting 48.6% top-1 on ImageNet-1K with ResNet-18 at IPC 10 (0.78% compression ratio), ahead of SRe2L, G-VBSM and RDED by 27.3, 17.2 and 6.6 points respectively.", "abstract": "Dataset condensation, a concept within data-centric learning, efficiently transfers critical attributes from an original dataset to a synthetic version, maintaining both diversity and realism. This approach significantly improves model training efficiency and is adaptable across multiple application areas. Previous methods in dataset condensation have faced challenges: some incur high computational costs which limit scalability to larger datasets (e.g., MTT, DREAM, and TESLA), while others are restricted to less optimal design spaces, which could hinder potential improvements, especially in smaller datasets (e.g., SRe2L, G-VBSM, and RDED). To address these limitations, we propose a comprehensive design framework that includes specific, effective strategies like implementing soft category-aware matching and adjusting the learning rate schedule. These strategies are grounded in empirical evidence and theoretical backing. Our resulting approach, Elucidate Dataset Condensation (EDC), establishes a benchmark for both small and large-scale dataset condensation. In our testing, EDC achieves state-of-the-art accuracy, reaching 48.6% on ImageNet-1k with a ResNet-18 model at an IPC of 10, which corresponds to a compression ratio of 0.78%. This performance exceeds those of SRe2L, G-VBSM, and RDED by margins of 27.3%, 17.2%, and 6.6%, respectively.", "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "scdd-2024", "type": "paper", "title": "Self-supervised Dataset Distillation: A Good Compression Is All You Need", "short": "SC-DD", "arxiv": "2404.07976", "url": null, "date": "2024-04-11", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Muxin Zhou", "Zeyuan Yin", "Shitong Shao", "Zhiqiang Shen" ], "kind": "method", "settings": [ "self-supervised" ], "families": [ "decoupled-synthesis" ], "tier": "catalogued", "links": { "code": "https://github.com/VILA-Lab/SRe2L/tree/main/SCDD/", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "SC-DD observes that as the pretrained model used for SRe2L-style inversion grows, supervised pretraining's batch-norm statistics flatten and become less informative for recovering data, whereas self-supervised pretrained models keep larger BN-statistic variance that yields stronger gradient signal during synthesis; it therefore performs decoupled inversion from self-supervised pretrained models instead of supervised ones. On CIFAR-100, Tiny-ImageNet and ImageNet-1K, the paper reports outperforming prior supervised decoupled-distillation methods (SRe2L, MTT, TESLA, DC, CAFE and others) by large margins when larger backbones are used, under the same recovery and post-training budgets.", "abstract": "Dataset distillation aims to compress information from a large-scale original dataset to a new compact dataset while striving to preserve the utmost degree of the original data informational essence. Previous studies have predominantly concentrated on aligning the intermediate statistics between the original and distilled data, such as weight trajectory, features, gradient, BatchNorm, etc. In this work, we consider addressing this task through the new lens of model informativeness in the compression stage on the original dataset pretraining. We observe that with the prior state-of-the-art SRe$^2$L, as model sizes increase, it becomes increasingly challenging for supervised pretrained models to recover learned information during data synthesis, as the channel-wise mean and variance inside the model are flatting and less informative. We further notice that larger variances in BN statistics from self-supervised models enable larger loss signals to update the recovered data by gradients, enjoying more informativeness during synthesis. Building on this observation, we introduce SC-DD, a simple yet effective Self-supervised Compression framework for Dataset Distillation that facilitates diverse information compression and recovery compared to traditional supervised learning schemes, further reaps the potential of large pretrained models with enhanced capabilities. Extensive experiments are conducted on CIFAR-100, Tiny-ImageNet and ImageNet-1K datasets to demonstrate the superiority of our proposed approach. The proposed SC-DD outperforms all previous state-of-the-art supervised dataset distillation methods when employing larger models, such as SRe$^2$L, MTT, TESLA, DC, CAFE, etc., by large margins under the same recovery and post-training budgets. Code is available at https://github.com/VILA-Lab/SRe2L/tree/main/SCDD/.", "builds_on": [ "sre2l-2023" ], "notes": "" }, { "id": "iid-2024", "type": "paper", "title": "Exploiting Inter-sample and Inter-feature Relations in Dataset Distillation", "short": "IID", "arxiv": "2404.00563", "url": null, "date": "2024-03-31", "venue": { "name": "CVPR 2024", "status": "conference" }, "authors": [ "Wenxiao Deng", "Wenbin Li", "Tianyu Ding", "Lei Wang", "Hongguang Zhang", "Kuihua Huang", "Jing Huo", "Yang Gao" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "tier": "catalogued", "links": { "code": "https://github.com/VincenDen/IID", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Adds a class-centralization constraint that pulls same-class synthetic features together and a covariance-matching constraint that aligns local feature covariance matrices, addressing dispersed within-class features and mean-only matching in prior distribution-matching methods, reporting up to 6.6 points on CIFAR-10 and 2.5-2.9 points on SVHN/CIFAR-100/Tiny-ImageNet over prior distribution-matching baselines.", "abstract": "Dataset distillation has emerged as a promising approach in deep learning, enabling efficient training with small synthetic datasets derived from larger real ones. Particularly, distribution matching-based distillation methods attract attention thanks to its effectiveness and low computational cost. However, these methods face two primary limitations: the dispersed feature distribution within the same class in synthetic datasets, reducing class discrimination, and an exclusive focus on mean feature consistency, lacking precision and comprehensiveness. To address these challenges, we introduce two novel constraints: a class centralization constraint and a covariance matching constraint. The class centralization constraint aims to enhance class discrimination by more closely clustering samples within classes. The covariance matching constraint seeks to achieve more accurate feature distribution matching between real and synthetic datasets through local feature covariance matrices, particularly beneficial when sample sizes are much smaller than the number of features. Experiments demonstrate notable improvements with these constraints, yielding performance boosts of up to 6.6% on CIFAR10, 2.9% on SVHN, 2.5% on CIFAR100, and 2.5% on TinyImageNet, compared to the state-of-the-art relevant methods. In addition, our method maintains robust performance in cross-architecture settings, with a maximum performance drop of 1.7% on four architectures. Code is available at https://github.com/VincenDen/IID.", "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ], "builds_on": [ "dm-2023", "idm-2023" ] }, { "id": "dilm-2024", "type": "paper", "title": "DiLM: Distilling Dataset into Language Model for Text-level Dataset Distillation", "short": "DiLM", "arxiv": "2404.00264", "url": null, "date": "2024-03-30", "venue": { "name": "NAACL 2024", "status": "conference" }, "authors": [ "Aru Maekawa", "Satoshi Kosugi", "Kotaro Funakoshi", "Manabu Okumura" ], "kind": "method", "settings": [ "text" ], "families": [ "generator-as-dataset", "gradient-matching" ], "facets": { "prior": [ "llm" ] }, "builds_on": [ "sldd-2021" ], "tier": "catalogued", "links": { "code": "https://github.com/arumaekawa/DiLM", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Proposes DiLM, which trains a language model to generate readable synthetic text examples for dataset distillation instead of optimizing word-embedding sequences directly, so the distilled data is not tied to one model's embedding table; reports it outperforms coreset-selection baselines on text classification and generalizes to training different model types and to in-context learning with large language models.", "abstract": "Dataset distillation aims to compress a training dataset by creating a small number of informative synthetic samples such that neural networks trained on them perform as well as those trained on the original training dataset. Current text dataset distillation methods create each synthetic sample as a sequence of word embeddings instead of a text to apply gradient-based optimization; however, such embedding-level distilled datasets cannot be used for training other models whose word embedding weights are different from the model used for distillation. To address this issue, we propose a novel text dataset distillation approach, called Distilling dataset into Language Model (DiLM), which trains a language model to generate informative synthetic training samples as text data, instead of directly optimizing synthetic samples. We evaluated DiLM on various text classification datasets and showed that distilled synthetic datasets from DiLM outperform those from current coreset selection methods. DiLM achieved remarkable generalization performance in training different types of models and in-context learning of large language models. Our code will be available at https://github.com/arumaekawa/DiLM.", "awesome_sections": [ "Applications / Text" ], "notes": "", "explained": { "before": "All prior text dataset distillation (TDD/sldd-2021's meta-learning approach, and attention-label distillation) optimizes synthetic examples as sequences of continuous word embeddings rather than discrete text, because gradient-based distillation objectives (originally designed for pixel-continuous images, e.g. DC's gradient matching) need a differentiable input.", "problem": "Embedding-level distilled datasets are tied to the exact embedding table of the model used for distillation and cannot train a different model; nearest-neighbor decoding of the embeddings back to words (tried by sldd-2021 and later work) produces unrelated, unreadable words, so embedding-level synthetic sets are neither model-agnostic nor interpretable.", "idea": "Instead of optimizing the discrete text or its embeddings directly, train the continuous parameters of a small generator language model (GPT-2) to emit informative synthetic text, backpropagating the gradient-matching loss into the generator's weights via a differentiable path (loss weighting by each sample's generation probability) that bypasses the non-differentiable sampling step.", "method": "Three stages: (1) pretrain the generator LM with a standard language-modeling loss on the real training data, using class-specific `` tokens so it learns to generate class-conditioned text; (2) fine-tune the generator with a nested bi-level loop — an outer loop that reinitializes a learner model (BERT-base) from $p(\\theta_0)$, an inner loop that updates the learner on real data for $K$ steps, and a per-inner-step DC-style gradient-matching loss $\\mathcal{L}_{GM}=D(\\nabla_\\theta \\mathcal{L}_{real},\\nabla_\\theta\\mathcal{L}_{syn})$ (cosine distance) computed per class, where $\\mathcal{L}_{syn}=\\sum_i a_i\\, l_\\theta(\\tilde{x}_i)$ weights each sampled synthetic example's learner loss by its normalized generation probability $a_i$ so gradients can flow into the generator $\\phi$; two add-on techniques improve this — a \"representative teacher\" (K-center clustering of real samples used as the real-data mini-batch instead of random sampling) and \"diverse mini-batch sampling\" (K-means clustering of an oversampled batch of generated candidates so each training step sees diverse synthetic samples); (3) generate the final distilled dataset by sampling 100x the target size via top-$p$ sampling and selecting K-center representative samples. Only the randomly initialized last layer of the learner is used in gradient matching to reduce cost. The stored artifact is the trained generator LM (or, for evaluation comparability, a fixed sampled text set).", "evidence": "Table 1 (SST-2/QQP/MNLI-m, BERT-base learner, hard labels, DPC=5/10/20): DiLM beats K-centers coreset selection at every setting (e.g. SST-2 DPC=20: 80.3% vs. 79.8%; MNLI-m DPC=20: 48.7% vs. 45.3%; full dataset 86.7%), and beats a \"Vanilla LM\" (generator without gradient-matching fine-tuning) by a wide margin (e.g. SST-2 DPC=5: 72.5% vs. 65.2%), isolating that gradient matching, not just LM-quality text generation, drives the gain. TDD (sldd-2021), evaluated as raw embeddings, beats DiLM and even the full dataset (SST-2 89.6%) but collapses to near-random (50.2%) once its embeddings are decoded to nearest- neighbor text — the paper's key demonstration that embedding-level distillation is not usable as text. Table 2 (cross-model generalization, DPC=20): DiLM-distilled text (generated using BERT-base) trains RoBERTa-base, BERT-large, and XLNet-base (a different architecture family) better than K-centers in 11/12 cells (e.g. SST-2 XLNet-base: 77.9% vs. 71.8%). Table 3 (5-shot in-context learning of SST-2): DiLM-selected examples beat Random and K-centers as ICL prompts across GPT-2-XL (1.5B), OPT (2.7B), and Llama 2 (7B) (e.g. Llama 2: 95.1% vs. 94.6% K-centers). Table 4 ablation (DPC=5): removing the representative-teacher, diverse-mini-batch-sampling, or final K-center selection component each individually drops accuracy (e.g. SST-2 drops from 72.5% to 65.2% without selection), confirming all three additions contribute.", "limitations": "A performance gap to the full dataset remains (e.g. SST-2 DPC=20 80.3% vs. 86.7% full); only classification tasks are tested, not generation or full LM pretraining; the small GPT-2 (128M) generator shows visible repetition artifacts on the harder MNLI-m task; training the generator adds upfront cost beyond directly optimizing embeddings; the paper explicitly notes DiLM cannot be used for privacy preservation, since the generator LM can memorize and regenerate real training samples (unlike grad-mm-2025's later explicit DP mechanism for the same discrete-text problem)." } }, { "id": "progressive-trajectory-matching-for-2024", "type": "paper", "title": "Progressive trajectory matching for medical dataset distillation", "arxiv": "2403.13469", "url": null, "date": "2024-03-20", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Zhen Yu", "Yang Liu", "Qingchao Chen" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "trajectory-matching" ], "applications": [ "medical" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Finds that plain trajectory matching (randomly matching parts of real training trajectories) is unstable and gives inferior results on medical images, so proposes a progressive trajectory-matching strategy for stability plus a dynamic overlap mitigation module that eliminates overlap across synthetic images and retrains parts of them to recover the diversity lost to the more stable matching. Reports 8.33% average improvement over prior state-of-the-art methods, and 11.7% at 2 images per class, on a new medical dataset distillation benchmark across multiple modalities.", "abstract": "It is essential but challenging to share medical image datasets due to privacy issues, which prohibit building foundation models and knowledge transfer. In this paper, we propose a novel dataset distillation method to condense the original medical image datasets into a synthetic one that preserves useful information for building an analysis model without accessing the original datasets. Existing methods tackle only natural images by randomly matching parts of the training trajectories of the model parameters trained by the whole real datasets. However, through extensive experiments on medical image datasets, the training process is extremely unstable and achieves inferior distillation results. To solve these barriers, we propose to design a novel progressive trajectory matching strategy to improve the training stability for medical image dataset distillation. Additionally, it is observed that improved stability prevents the synthetic dataset diversity and final performance improvements. Therefore, we propose a dynamic overlap mitigation module that improves the synthetic dataset diversity by dynamically eliminating the overlap across different images and retraining parts of the synthetic images for better convergence. Finally, we propose a new medical image dataset distillation benchmark of various modalities and configurations to promote fair evaluations. It is validated that our proposed method achieves 8.33% improvement over previous state-of-the-art methods on average, and 11.7% improvement when ipc=2 (i.e., image per class is 2). Codes and benchmarks will be released.", "awesome_sections": [ "Applications / Medical" ], "notes": "" }, { "id": "dd-robustbench-2025", "type": "paper", "title": "DD-RobustBench: An Adversarial Robustness Benchmark for Dataset Distillation", "short": "DD-RobustBench", "arxiv": "2403.13322", "url": null, "date": "2024-03-20", "venue": { "name": "TIP 2025", "status": "journal" }, "authors": [ "Yifan Wu", "Jiawei Du", "Ping Liu", "Yuewei Lin", "Wei Xu", "Wenqing Cheng" ], "kind": "evaluation-benchmark", "settings": [ "image-classification" ], "families": [], "applications": [ "robustness" ], "tier": "catalogued", "links": { "code": "https://github.com/FredWU-HUST/DD-RobustBench", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Extends adversarial-robustness benchmarking of distilled datasets to a wider range of methods (including TESLA and SRe2L) and to ImageNet-1K, evaluating against PGD and AutoAttack and analyzing robustness from a frequency perspective; finds that mixing distilled data into standard training batches on the original dataset can improve robustness.", "abstract": "Dataset distillation is an advanced technique aimed at compressing datasets into significantly smaller counterparts, while preserving formidable training performance. Significant efforts have been devoted to promote evaluation accuracy under limited compression ratio while overlooked the robustness of distilled dataset. In this work, we introduce a comprehensive benchmark that, to the best of our knowledge, is the most extensive to date for evaluating the adversarial robustness of distilled datasets in a unified way. Our benchmark significantly expands upon prior efforts by incorporating a wider range of dataset distillation methods, including the latest advancements such as TESLA and SRe2L, a diverse array of adversarial attack methods, and evaluations across a broader and more extensive collection of datasets such as ImageNet-1K. Moreover, we assessed the robustness of these distilled datasets against representative adversarial attack algorithms like PGD and AutoAttack, while exploring their resilience from a frequency perspective. We also discovered that incorporating distilled data into the training batches of the original dataset can yield to improvement of robustness.", "awesome_sections": [ "Main / Benchmark" ], "notes": "" }, { "id": "guard-2025", "type": "paper", "title": "Towards Adversarially Robust Dataset Distillation by Curvature Regularization", "short": "GUARD", "arxiv": "2403.10045", "url": null, "date": "2024-03-15", "venue": { "name": "AAAI 2025", "status": "conference" }, "authors": [ "Eric Xue", "Yijiang Li", "Haoyang Liu", "Peiran Wang", "Yifan Shen", "Haohan Wang" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "applications": [ "adversarial-robustness" ], "tier": "notable", "links": { "code": "https://github.com/yumozi/GUARD", "project": "https://yumozi.github.io/GUARD/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Adds curvature regularization to the distillation process so that models trained on the resulting distilled set are adversarially robust without the cost of full adversarial training. Reports outperforming standard adversarial training on both accuracy and robustness at lower computational overhead, and withstanding various adversarial attacks.", "abstract": "Dataset distillation (DD) allows datasets to be distilled to fractions of their original size while preserving the rich distributional information, so that models trained on the distilled datasets can achieve a comparable accuracy while saving significant computational loads. Recent research in this area has been focusing on improving the accuracy of models trained on distilled datasets. In this paper, we aim to explore a new perspective of DD. We study how to embed adversarial robustness in distilled datasets, so that models trained on these datasets maintain the high accuracy and meanwhile acquire better adversarial robustness. We propose a new method that achieves this goal by incorporating curvature regularization into the distillation process with much less computational overhead than standard adversarial training. Extensive empirical experiments suggest that our method not only outperforms standard adversarial training on both accuracy and robustness with less computation overhead but is also capable of generating robust distilled datasets that can withstand various adversarial attacks. Our implementation is available at: https://github.com/yumozi/GUARD.", "awesome_sections": [ "Applications / Robustness" ], "notes": "" }, { "id": "timedc-2024", "type": "paper", "title": "Dataset Condensation for Time Series Classification via Dual Domain Matching", "short": "CondTSC", "arxiv": "2403.07245", "url": null, "date": "2024-03-12", "venue": { "name": "KDD 2024", "status": "conference" }, "authors": [ "Zhanyu Liu", "Ke Hao", "Guanjie Zheng", "Yanwei Yu" ], "kind": "method", "settings": [ "time-series" ], "families": [ "trajectory-matching", "distribution-matching" ], "builds_on": [ "mtt-2022", "dm-2023", "dc-2021" ], "tier": "catalogued", "links": { "code": "https://github.com/zhyliu00/TimeSeriesCond", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "CondTSC adapts image-style dataset condensation to time series classification with dual surrogate matching objectives computed in both the time and frequency domains (rather than time domain alone), combined with multi-view data augmentation and dual-domain training, to better capture the temporal and spectral structure that direct adaptations of image condensation methods miss. The paper reports outperforming other time-series and adapted image/graph condensation baselines, producing a condensed set that better matches the real data's distribution.", "abstract": "Time series data has been demonstrated to be crucial in various research fields. The management of large quantities of time series data presents challenges in terms of deep learning tasks, particularly for training a deep neural network. Recently, a technique named \\textit{Dataset Condensation} has emerged as a solution to this problem. This technique generates a smaller synthetic dataset that has comparable performance to the full real dataset in downstream tasks such as classification. However, previous methods are primarily designed for image and graph datasets, and directly adapting them to the time series dataset leads to suboptimal performance due to their inability to effectively leverage the rich information inherent in time series data, particularly in the frequency domain. In this paper, we propose a novel framework named Dataset \\textit{\\textbf{Cond}}ensation for \\textit{\\textbf{T}}ime \\textit{\\textbf{S}}eries \\textit{\\textbf{C}}lassification via Dual Domain Matching (\\textbf{CondTSC}) which focuses on the time series classification dataset condensation task. Different from previous methods, our proposed framework aims to generate a condensed dataset that matches the surrogate objectives in both the time and frequency domains. Specifically, CondTSC incorporates multi-view data augmentation, dual domain training, and dual surrogate objectives to enhance the dataset condensation process in the time and frequency domains. Through extensive experiments, we demonstrate the effectiveness of our proposed framework, which outperforms other baselines and learns a condensed synthetic dataset that exhibits desirable characteristics such as conforming to the distribution of the original data.", "notes": "Resolved after full-text read: the \"dual surrogate objectives\" are (1) multi-step gradient matching (an MTT-style trajectory-matching loss, Eq. 9-11) and (2) parameter-space/embedding matching (a DM-style distribution-matching loss on trained-parameter embeddings rather than random-network embeddings, Eq. 12-15), each computed once in the time domain and once in the frequency domain.", "explained": { "before": "Dataset condensation for time series had not been systematically studied; the few applicable methods were image/graph condensation techniques (DD, DC, DSA, DM, MTT, IDC, HaBa) applied unmodified to raw time-domain sequences, using image-style random-pixel initialization and image-style augmentations (jittering, cropping).", "problem": "Directly porting image/graph condensation to time series performs poorly because it only uses the time domain, ignoring that time series have periodicity/seasonality structure that is far more separable in the frequency domain (confirmed by the paper's own t-SNE visualization, Fig. 3, showing better class separation in frequency space); it also inherits image-style random-pixel initialization, which the paper finds empirically performs poorly for time series.", "idea": "Match the same surrogate objectives (multi-step gradient matching, parameter-space embedding matching) used by prior image methods, but compute them in both the time domain and the Fourier-transformed frequency domain, and enrich the synthetic set at each step with additional frequency-based augmented views (low-pass filtering, phase perturbation, magnitude perturbation) so the matching signal draws on frequency structure the time domain alone does not expose.", "method": "Initialize the synthetic set $\\mathcal{S}$ via K-means cluster centroids per class (not random samples, per the empirical finding above). At each training epoch, for each of four augmented views (raw, LPF, phase-perturbed, magnitude-perturbed — Eq. 5-8, sequentially applied via FFT/inverse-FFT), compute both the time-domain and frequency-domain (via FFT) versions of the view; train a network on $\\mathcal{S}$ for $N{=}10$ steps and on real data $\\mathcal{T}$ for $M{=}1000$ steps from the same sampled initialization $\\theta_0\\sim p(\\theta)$ (approximated by real-data training trajectories, as in MTT), separately in each domain, to get $\\hat\\theta_N$ and $\\bar\\theta_M$; compute a multi-step gradient- matching loss $\\mathcal{L}^{grad}=\\|\\hat\\theta_N-\\bar\\theta_M\\|^2/\\|\\theta_0-\\bar\\theta_M\\|^2$ (Eq. 9-11, an MTT-style trajectory-matching term) and a parameter-space embedding-matching loss $\\mathcal{L}^{emb}=\\|\\text{Mean}(\\hat{H}_N)-\\text{Mean}(\\bar{H}_M)\\|^2$ (Eq. 12-15, a DM-style distribution-matching term, but computed on embeddings from the trained trajectory parameters rather than random-network embeddings) in each domain; sum all four views' losses ($\\mathcal{L}=\\mathcal{L}_{raw}+\\mathcal{L}_{LPF}+\\mathcal{L}_{PP}+\\mathcal{L}_{MP}$, each itself $\\mathcal{L}_t^{grad}+\\mathcal{L}_f^{grad}+\\lambda(\\mathcal{L}_t^{emb}+\\mathcal{L}_f^{emb})$) and backpropagate to update $\\mathcal{S}$. Stored artifact: only the time-domain synthetic sequences and hard labels (frequency-domain versions and augmented views exist only during synthesis).", "evidence": "Table 2 (5 datasets — HAR, Electric, Insect, FD, Sleep — CNN+BN evaluator, hard labels, 10 baselines including DD/DC/DSA/DM/MTT/IDC/HaBa and coreset methods): CondTSC beats every baseline at every spc/ratio with statistical significance (t-test p<0.01), e.g. HAR spc=1 (0.1%): 61.38% vs. best baseline (DC) 58.30% (full data 93.14%); Insect spc=1 (0.05%): 45.15% vs. best baseline (K-means) 16.70% (full 70.78%) — the largest margin in the paper, on the dataset where frequency structure matters most; FD spc=20 (0.72%): 90.78% vs. best baseline (MTT) 74.63% (full 98.51%). NAS downstream task (Table 3, HAR, 324-ConvNet search space, 1% ratio): CondTSC's condensed-set accuracy ranking correlates best with full-data ranking (Spearman 0.665 vs. next-best MTT 0.582) at comparable time cost (~24.5 min for all methods). Ablation (Table 4, Base=MTT-only, +A=augmentation, +T=dual-domain training, +M=dual-objective matching): each module adds accuracy monotonically; the Dual Domain Training module (+T) gives the largest jump on frequency-friendly datasets (Insect spc=1: 18.58%→44.64%, FD spc=1: 51.41%→69.62%), directly isolating the frequency domain as the dominant source of CondTSC's advantage over MTT. Initialization ablation (Table 5): K-means initialization outperforms random, but CondTSC still substantially improves every initialization scheme tested (11-26% relative gains), showing the method is not solely reliant on its K-means initialization choice.", "limitations": "Evaluation network is limited to a single CNN+BN architecture in the main results (cross-architecture results are relegated to an appendix); only classification is addressed, not forecasting (the complementary time-series problem condtsf-2024 targets); the frequency-domain augmentations (LPF, phase/ magnitude perturbation) are hand-designed and dataset-agnostic hyperparameters (e.g. 50% low-pass cutoff) rather than learned or tuned per dataset; cost is reported only as aggregate wall-clock in the NAS experiment (~24.5 min, comparable to baselines) without a breakdown of the added frequency-domain computation's overhead." } }, { "id": "graph-data-condensation-via-2024", "type": "paper", "title": "Graph Data Condensation via Self-expressive Graph Structure Reconstruction", "arxiv": "2403.07294", "url": null, "date": "2024-03-12", "venue": { "name": "KDD 2024", "status": "conference" }, "authors": [ "Zhanyu Liu", "Chaolv Zeng", "Guanjie Zheng" ], "kind": "method", "settings": [ "graph" ], "families": [ "gradient-matching" ], "facets": { "parameterization": [ "graph-structure" ] }, "tier": "catalogued", "links": { "code": "https://github.com/zclzcl0223/GCSR", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "GCSR (GCSR) condenses a graph by reconstructing an interpretable, closed-form self-expressive adjacency for the condensed nodes (each node reconstructed from a sparse combination of the others in its feature subspace) instead of learning a structure generator or dropping structure entirely, while node features are refined by multi-step gradient matching against the full graph's training trajectories. The paper reports the best average cross-architecture GNN accuracy among graph condensation baselines, attributing the gain to keeping explicit structural information.", "abstract": "With the increasing demands of training graph neural networks (GNNs) on large-scale graphs, graph data condensation has emerged as a critical technique to relieve the storage and time costs during the training phase. It aims to condense the original large-scale graph to a much smaller synthetic graph while preserving the essential information necessary for efficiently training a downstream GNN. However, existing methods concentrate either on optimizing node features exclusively or endeavor to independently learn node features and the graph structure generator. They could not explicitly leverage the information of the original graph structure and failed to construct an interpretable graph structure for the synthetic dataset. To address these issues, we introduce a novel framework named \\textbf{G}raph Data \\textbf{C}ondensation via \\textbf{S}elf-expressive Graph Structure \\textbf{R}econstruction (\\textbf{GCSR}). Our method stands out by (1) explicitly incorporating the original graph structure into the condensing process and (2) capturing the nuanced interdependencies between the condensed nodes by reconstructing an interpretable self-expressive graph structure. Extensive experiments and comprehensive analysis validate the efficacy of the proposed method across diverse GNN models and datasets. Our code is available at \\url{https://github.com/zclzcl0223/GCSR}.", "builds_on": [ "gcond-2022" ], "notes": "Family assigned from secondary sources (paper text/GitHub description via search), since the abstract alone does not name the matching mechanism." }, { "id": "mdc-2024", "type": "paper", "title": "Multisize Dataset Condensation", "short": "MDC", "arxiv": "2403.06075", "url": null, "date": "2024-03-10", "venue": { "name": "ICLR 2024", "status": "conference" }, "authors": [ "Yang He", "Lingao Xiao", "Joey Tianyi Zhou", "Ivor Tsang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/he-y/Multisize-Dataset-Condensation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Compresses N separate condensation runs at N target sizes into a single condensation process by adding an adaptive subset loss that keeps any prefix subset of the condensed images representative on its own, addressing the 'subset degradation problem' where a subset of a larger condensed set underperforms a set condensed directly at that smaller size; reports 5.22-6.40 point average accuracy gains condensing CIFAR-10 to 10 images per class.", "abstract": "While dataset condensation effectively enhances training efficiency, its application in on-device scenarios brings unique challenges. 1) Due to the fluctuating computational resources of these devices, there's a demand for a flexible dataset size that diverges from a predefined size. 2) The limited computational power on devices often prevents additional condensation operations. These two challenges connect to the \"subset degradation problem\" in traditional dataset condensation: a subset from a larger condensed dataset is often unrepresentative compared to directly condensing the whole dataset to that smaller size. In this paper, we propose Multisize Dataset Condensation (MDC) by compressing N condensation processes into a single condensation process to obtain datasets with multiple sizes. Specifically, we introduce an \"adaptive subset loss\" on top of the basic condensation loss to mitigate the \"subset degradation problem\". Our MDC method offers several benefits: 1) No additional condensation process is required; 2) reduced storage requirement by reusing condensed images. Experiments validate our findings on networks including ConvNet, ResNet and DenseNet, and datasets including SVHN, CIFAR-10, CIFAR-100 and ImageNet. For example, we achieved 5.22%-6.40% average accuracy gains on condensing CIFAR-10 to ten images per class. Code is available at: https://github.com/he-y/Multisize-Dataset-Condensation.", "awesome_sections": [ "Main / Better Optimization" ] }, { "id": "ld3m-2025", "type": "paper", "title": "Unlocking Dataset Distillation with Diffusion Models", "short": "LD3M", "arxiv": "2403.03881", "url": null, "date": "2024-03-06", "venue": { "name": "NeurIPS 2025", "status": "conference" }, "authors": [ "Brian B. Moser", "Federico Raue", "Sebastian Palacio", "Stanislav Frolov", "Andreas Dengel" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "generative-latent-optimization" ], "facets": { "parameterization": [ "generative-latent" ], "labels": "hard", "prior": [ "diffusion" ], "regime": "single-level", "scale": [ "imagenet-subsets" ] }, "tier": "notable", "links": { "code": "https://github.com/Brian-Moser/prune_and_distill", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "The first method to learn gradient-based distilled latents and class embeddings end-to-end through a pretrained latent diffusion model, rather than only sampling from a fixed prior or relying on GANs/autoencoders; a linearly decaying skip connection injected at every reverse step preserves gradient signal across dozens of timesteps without fine-tuning the diffusion weights, improving downstream accuracy by up to 4.8 points (1 IPC) and 4.2 points (10 IPC) over the prior state of the art on ImageNet subsets at 128x128 and 256x256.", "abstract": "Dataset distillation seeks to condense datasets into smaller but highly representative synthetic samples. While diffusion models now lead all generative benchmarks, current distillation methods avoid them and rely instead on GANs or autoencoders, or, at best, sampling from a fixed diffusion prior. This trend arises because naive backpropagation through the long denoising chain leads to vanishing gradients, which prevents effective synthetic sample optimization. To address this limitation, we introduce Latent Dataset Distillation with Diffusion Models (LD3M), the first method to learn gradient-based distilled latents and class embeddings end-to-end through a pre-trained latent diffusion model. A linearly decaying skip connection, injected from the initial noisy state into every reverse step, preserves the gradient signal across dozens of timesteps without requiring diffusion weight fine-tuning. Across multiple ImageNet subsets at 128x128 and 256x256, LD3M improves downstream accuracy by up to 4.8 percentage points (1 IPC) and 4.2 points (10 IPC) over the prior state-of-the-art. The code for LD3M is provided at https://github.com/Brian-Moser/prune_and_distill.", "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "notes": "Reclassified from diffusion-guided-synthesis to generative-latent-optimization: LD3M backpropagates a matching-style loss through a frozen pretrained diffusion model's latents/embeddings (no dataset-specific fine-tuning, no plain sampling), matching this family's formulation exactly (G frozen, latent-diffusion is one of its named instances per docs/taxonomy.md).", "explained": { "before": "GLaD backpropagates gradient/distribution/trajectory-matching losses through a frozen StyleGAN-XL's latents, but needs costly GAN inversion for initialization and its W+ multi-level latent space is expensive to optimize; sampling-based diffusion methods (Minimax Diffusion, D4M) avoid backpropagating through the denoising chain entirely by selecting or fine-tuning toward representative samples; an autoencoder-only approach (Duan et al., 2023) optimizes latents decoded by an LDM's autoencoder but ignores the diffusion process itself.", "problem": "Naive backpropagation of a matching loss through the T-step reverse diffusion chain to the initial latent $\\mathcal{Z}$ suffers vanishing gradients: the paper measures gradient norms for $\\mathcal{Z}$ falling roughly tenfold as the number of diffusion steps $T$ increases from 10 to 90 (Table 1), which the paper attributes to the product of T bounded Jacobians in the chain rule (Eq. 1) and which has forced every prior diffusion-based distillation method to avoid true end-to-end latent optimization.", "idea": "Inject a residual/skip connection from the fixed initial noisy state $\\mathbf{z}_T$ into every reverse step, with a linearly decaying weight from 1 (at $t=T$) to ~0 (as $t\\to0$), so gradients from the distillation loss have a short path back to the learnable latent $\\mathcal{Z}$ that bypasses most of the Jacobian chain, without ever fine-tuning the diffusion model's own weights.", "method": "A pretrained ImageNet-trained LDM (2x-compression autoencoder $\\mathcal{E}/\\mathcal{D}$, frozen U-Net denoiser $\\mu_\\theta$) is used without fine-tuning. Learnable latent codes $\\mathcal{Z}$ (initialized by encoding a small set of randomly-selected real images via $\\mathcal{E}$, not by GAN-style inversion) and learnable class-conditioning codes $\\mathbf{c}$ (initialized from the pretrained class-embedding network) are optimized. The modified reverse step $\\mathbf{z}_{t-1}\\leftarrow((1-t/T)\\mu_\\theta(\\mathbf{c},\\mathbf{z}_t,\\gamma_t)+(t/T)\\mathbf{z}_T)+\\sigma_t^2\\varepsilon_t$ (Eq. 7) replaces the standard predicted mean with a decaying blend of the model's prediction and the fixed $\\mathbf{z}_T$; the final $\\mathbf{z}_0$ is decoded to $\\mathcal{S}=\\mathcal{D}(\\mathbf{z}_0)$ and plugged into an existing distillation objective (DC, DM, or MTT) exactly as GLaD does for StyleGAN-XL, with the resulting loss backpropagated through the whole modified chain into $\\mathcal{Z}$ and $\\mathbf{c}$. Gradient checkpointing (as in GLaD) manages VRAM by recomputing forward segments during the backward pass rather than storing the full graph. Default diffusion steps are $T{=}10$ (128x128) or $T{=}20$ (256x256).", "evidence": "CIFAR-10 IPC=1, cross-arch avg. over AlexNet/ResNet18/VGG11/ViT, hard labels (Table 2a): LD3M(DC) 28.6% vs. GLaD(trained G) 26.3% vs. pixel-space DC 26.0%; LD3M(DM) 24.9% vs. GLaD(DM) 23.8%; sampling-based D4M reported ~10% under this same protocol. CIFAR-10 IPC=50 ConvNet (Table 2b, note differing optimization/IPC across rows): LD3M 73.2% vs. D4M 72.8% vs. decoupled SRe2L 60.2% (SRe2L's number is from its own IPC=1K report). ImageNet 10-class subsets, 128x128, IPC=1, avg. over 10 subsets and 4 archs (Table 3): LD3M improves over GLaD by +3.76% (DC), and over GLaD DM by roughly +16.3 points relative to pixel-space DM; at IPC=10 (Table 4, ImageNet A-E) LD3M beats GLaD by +2.52% (DC) and +3.46% (DM), reaching 47.1-47.3% average. At 256x256, IPC=1 (Table 5) LD3M beats GLaD by ~+2pp averaged across three generator-pretraining sources (ImageNet/FFHQ/Random) and by ~+7pp over pixel space. Cost: at T=20 on an A100-40GB, LD3M uses slightly less peak memory than GLaD (29.4GB vs. 31.2GB) and is faster (574 vs. 693 min total distillation). Key ablations: (1) Table 6 isolates the diffusion process itself — removing it (autoencoder-only, akin to Duan et al.) drops DC/IPC=1 ImageNet-A-E average from 36.5% to 35.3%, statistically similar to GLaD's 35.4%, showing the diffusion prior alone (without the gradient fix) adds nothing; (2) Table 7 isolates the gradient-flow fix — learnable conditioning alone gives 15.8% avg (ImageNette, MTT, IPC=1), adding learnable $\\mathcal{Z}$ raises it to 22.3% (still below GLaD's 26.6%), and only adding the decaying residual (Eq. 7) reaches 28.1%, confirming the skip connection, not latent learnability alone, is what makes diffusion-based end-to-end optimization competitive; (3) Table 8 shows real-image latent initialization vastly outperforms Gaussian-noise initialization for DC/DM (e.g. DC ImageNette 32.9% vs. 13.1%), while MTT is comparatively robust to initialization.", "limitations": "The LDM used is trained on ImageNet, i.e. matched to the target distribution for the ImageNet-subset experiments (only the 256x256 table separately tests FFHQ- and randomly-initialized generators, and even there the gap between ImageNet/FFHQ/Random pretraining is small, similarly to GLaD's own out-of-distribution-generator finding); the paper's own stated limitation (Appendix E) is that the linear decay schedule for the residual may not fully solve vanishing gradients at larger T, and alternative (non-linear) decay schedules are left untested; evaluated only on CIFAR-10 and 10-class ImageNet subsets up to 256x256, not full ImageNet-1K (explicitly named as future work, \"contingent on computational resources\"); hard labels only, single RTX A6000/A100 hardware; the SRe2L/D4M numbers in Table 2b are copied from their original papers under different IPC/optimization protocols, flagged by the paper itself as not a like-for-like comparison." } }, { "id": "improve-cross-architecture-generalization-2024", "type": "paper", "title": "Improve Cross-Architecture Generalization on Dataset Distillation", "short": "Model Pool", "arxiv": "2402.13007", "url": null, "date": "2024-02-20", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Binglin Zhou", "Linhao Zhong", "Wentao Chen" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Selects distillation models from a diverse pool according to a probability distribution during synthesis, rather than optimizing against one fixed network, so the distilled images are less biased toward a single architecture, and applies knowledge distillation when evaluating the distilled set. Reports improved test accuracy across a range of evaluation architectures compared to single-model distillation baselines.", "abstract": "Dataset distillation, a pragmatic approach in machine learning, aims to create a smaller synthetic dataset from a larger existing dataset. However, existing distillation methods primarily adopt a model-based paradigm, where the synthetic dataset inherits model-specific biases, limiting its generalizability to alternative models. In response to this constraint, we propose a novel methodology termed \"model pool\". This approach involves selecting models from a diverse model pool based on a specific probability distribution during the data distillation process. Additionally, we integrate our model pool with the established knowledge distillation approach and apply knowledge distillation to the test process of the distilled dataset. Our experimental results validate the effectiveness of the model pool approach across a range of existing models while testing, demonstrating superior performance compared to existing methodologies.", "notes": "" }, { "id": "two-trades-is-not-2024", "type": "paper", "title": "Two Trades is not Baffled: Condensing Graph via Crafting Rational Gradient Matching", "short": "CTRL", "arxiv": "2402.04924", "url": null, "date": "2024-02-07", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Tianle Zhang", "Yuchen Zhang", "Kun Wang", "Kai Wang", "Beining Yang", "Kaipeng Zhang", "Wenqi Shao", "Ping Liu", "Joey Tianyi Zhou", "Yang You" ], "kind": "method", "settings": [ "graph" ], "families": [ "gradient-matching" ], "tier": "catalogued", "links": { "code": "https://github.com/nus-hpc-ai-lab/ctrl", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "CTRL observes that prior graph condensation methods match only the direction of gradients, which lets the condensed graph's training trajectory drift from the real graph's and accumulates error (worsened by condensation/evaluation-phase mismatch); it corrects this with an optimized starting point closer to the real feature distribution and a more refined gradient-matching strategy. The paper reports that CTRL neutralizes the impact of accumulated errors on condensed-graph performance, with experiments across multiple graph datasets and downstream tasks.", "abstract": "Training on large-scale graphs has achieved remarkable results in graph representation learning, but its cost and storage have raised growing concerns. As one of the most promising directions, graph condensation methods address these issues by employing gradient matching, aiming to condense the full graph into a more concise yet information-rich synthetic set. Though encouraging, these strategies primarily emphasize matching directions of the gradients, which leads to deviations in the training trajectories. Such deviations are further magnified by the differences between the condensation and evaluation phases, culminating in accumulated errors, which detrimentally affect the performance of the condensed graphs. In light of this, we propose a novel graph condensation method named \\textbf{C}raf\\textbf{T}ing \\textbf{R}ationa\\textbf{L} trajectory (\\textbf{CTRL}), which offers an optimized starting point closer to the original dataset's feature distribution and a more refined strategy for gradient matching. Theoretically, CTRL can effectively neutralize the impact of accumulated errors on the performance of condensed graphs. We provide extensive experiments on various graph datasets and downstream tasks to support the effectiveness of CTRL. Code is released at https://github.com/NUS-HPC-AI-Lab/CTRL.", "builds_on": [ "gcond-2022" ], "notes": "" }, { "id": "group-distributionally-robust-dataset-2025", "type": "paper", "title": "Group Distributionally Robust Dataset Distillation with Risk Minimization", "arxiv": "2402.04676", "url": null, "date": "2024-02-07", "venue": { "name": "ICLR 2025", "status": "conference" }, "authors": [ "Saeed Vahidian", "Mingyu Wang", "Jianyang Gu", "Vyacheslav Kungurtsev", "Wei Jiang", "Yiran Chen" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "applications": [ "robustness", "fairness" ], "tier": "notable", "links": { "code": "https://github.com/Mming11/RobustDatasetDistillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Argues that matching training-set convergence, the usual DD criterion, does not guarantee that a model trained on the synthetic set generalizes to under-represented subgroups, since coverage of low-density regions matters more than training error there; proposes combining clustering with minimization of a distributionally-robust risk measure on the loss during distillation. Reports a theoretical rationale and improved generalization and robustness across subgroups compared to standard DD.", "abstract": "Dataset distillation (DD) has emerged as a widely adopted technique for crafting a synthetic dataset that captures the essential information of a training dataset, facilitating the training of accurate neural models. Its applications span various domains, including transfer learning, federated learning, and neural architecture search. The most popular methods for constructing the synthetic data rely on matching the convergence properties of training the model with the synthetic dataset and the training dataset. However, using the empirical loss as the criterion must be thought of as auxiliary in the same sense that the training set is an approximate substitute for the population distribution, and the latter is the data of interest. Yet despite its popularity, an aspect that remains unexplored is the relationship of DD to its generalization, particularly across uncommon subgroups. That is, how can we ensure that a model trained on the synthetic dataset performs well when faced with samples from regions with low population density? Here, the representativeness and coverage of the dataset become salient over the guaranteed training error at inference. Drawing inspiration from distributionally robust optimization, we introduce an algorithm that combines clustering with the minimization of a risk measure on the loss to conduct DD. We provide a theoretical rationale for our approach and demonstrate its effective generalization and robustness across subgroups through numerical experiments. The source code is available at https://github.com/Mming11/RobustDatasetDistillation.", "awesome_sections": [ "Applications / Robustness" ], "notes": "" }, { "id": "geom-2024", "type": "paper", "title": "Navigating Complexity: Toward Lossless Graph Condensation via Expanding Window Matching", "short": "GEOM", "arxiv": "2402.05011", "url": null, "date": "2024-02-07", "venue": { "name": "ICML 2024", "status": "conference" }, "authors": [ "Yuchen Zhang", "Tianle Zhang", "Kai Wang", "Ziyao Guo", "Yuxuan Liang", "Xavier Bresson", "Wei Jin", "Yang You" ], "kind": "method", "settings": [ "graph" ], "families": [ "trajectory-matching" ], "facets": { "parameterization": [ "other" ], "labels": "mixed", "prior": [ "trained-experts" ], "regime": "bilevel", "scale": [ "large-scale-other" ] }, "builds_on": [ "sfgc-2023" ], "tier": "notable", "links": { "code": "https://github.com/nus-hpc-ai-lab/geom", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Identifies that the prior state-of-the-art graph trajectory-matching method supplies biased and restricted supervision signals from the original graph, limiting both the scale and efficacy of the condensed graph, and proposes GEOM, which trains expert trajectories with a curriculum-learning strategy for more diverse supervision and transfers that information into the condensed graph with an expanding-window matching scheme; reports state-of-the-art results toward lossless graph condensation across several datasets.", "abstract": "Graph condensation aims to reduce the size of a large-scale graph dataset by synthesizing a compact counterpart without sacrificing the performance of Graph Neural Networks (GNNs) trained on it, which has shed light on reducing the computational cost for training GNNs. Nevertheless, existing methods often fall short of accurately replicating the original graph for certain datasets, thereby failing to achieve the objective of lossless condensation. To understand this phenomenon, we investigate the potential reasons and reveal that the previous state-of-the-art trajectory matching method provides biased and restricted supervision signals from the original graph when optimizing the condensed one. This significantly limits both the scale and efficacy of the condensed graph. In this paper, we make the first attempt toward \\textit{lossless graph condensation} by bridging the previously neglected supervision signals. Specifically, we employ a curriculum learning strategy to train expert trajectories with more diverse supervision signals from the original graph, and then effectively transfer the information into the condensed graph with expanding window matching. Moreover, we design a loss function to further extract knowledge from the expert trajectories. Theoretical analysis justifies the design of our method and extensive experiments verify its superiority across different datasets. Code is released at https://github.com/NUS-HPC-AI-Lab/GEOM.", "awesome_sections": [ "Applications / Graph Neural Network" ], "notes": "", "explained": { "before": "SFGC introduced structure-free graph condensation via short-segment training-trajectory meta-matching against offline expert trajectories, improving on GCond's short-range online gradient matching. GEOM diagnoses a specific remaining failure of SFGC-style trajectory matching on large graphs: performance saturates as the condensation ratio grows and never reaches the original graph's accuracy.", "problem": "GEOM traces the saturation to the composition of the supervision signal: visualizing gradient norms by node \"difficulty\" (homophily-based), it finds difficult (low-homophily) nodes dominate the expert trajectory's gradient, biasing the condensed graph toward difficult-node patterns and away from the easy, representative nodes that actually carry most of the class signal; separately, prior trajectory matching only samples a narrow, fixed range of expert checkpoints, wasting most of the trajectory.", "idea": "Fix the supervision at its source with two changes: (1) train expert trajectories with curriculum learning (easy-to-difficult node ordering by a homophily-based difficulty score) so trajectories carry a better mix of easy/difficult supervision, and (2) match against an expanding window of expert checkpoints (growing from an early, narrow range to a wide one over the course of condensation) instead of a fixed or sliding window, using ~10x more checkpoints than prior trajectory matching; a theoretical accumulated-error decomposition (initialization error + matching error, following Du et al. 2023) justifies why both changes reduce the training/evaluation-time distribution gap.", "method": "Buffer phase: train GNN experts on the full graph with a curriculum scheduler (nodes ranked by a neighborhood-label-entropy difficulty score, easy nodes introduced first via a pacing function) and save checkpoints $\\{\\theta_t^*\\}$. Condensation phase: sample a checkpoint pair within an expanding matching range $\\mathcal{R}$ (bounded by growing thresholds $U,U'$), initialize a student GNN$_\\mathcal{S}$ at $\\theta_t^*$, unroll $q$ training steps on the structure-free condensed data $\\mathcal{S}=(\\widetilde{ \\mathbf{X}},\\mathbf{I},\\widetilde{\\mathbf{Y}})$ with a learnable learning rate, and minimize a trajectory meta-matching loss $\\mathcal{L}_M$ (same form as SFGC) plus a Knowledge-Embedding-Extractor term $\\mathcal{L}_E=\\mathrm{KL}(f(\\theta_T^*;\\mathcal{S})\\,\\|\\,\\widetilde{\\mathcal{Y}})$ that KL-distills soft labels $\\widetilde{\\mathcal{Y}}$ from late-trajectory expert checkpoints into the condensed data, backpropagating both terms through the $q$-step unroll to update $\\widetilde{\\mathbf{X}}$ and $\\widetilde{\\mathcal{Y}}$.", "evidence": "Table 1 (GCN condensation/evaluation, node classification, 10 repeats): state-of-the-art in 18/19 settings; reports the first \"lossless\" graph condensation results — ogbn-arxiv r=5% reaches 71.4% vs. 71.4% whole dataset (exact match, where GCond ran out of memory at this ratio); Cora r=2.6% reaches 83.6% vs. 81.2% whole (exceeding the original graph); Citeseer r=1.8% reaches 74.3% vs. 71.7% whole. Table 2 (cross-architecture, MLP/GAT/APPNP/Cheby/GCN/SAGE/SGC): lossless in 20/35 architecture x dataset x ratio cells. Key ablation (Table 3): expanding-window matching beats two fixed-window variants and a sliding-window variant on both Cora and Citeseer, with or without curriculum learning (CL); CL alone gives smaller gains than expanding-window alone, but the combination is best (e.g. Cora 80.8->83.6 sliding-w/o-CL to expanding-with-CL); a separate ablation (Fig. 4) shows the KEE soft-label term gives further gains especially once the trajectory-matching loss plateaus.", "limitations": "Explicitly stated: GEOM still requires deriving expert trajectories in advance, adding non-trivial upfront GNN-training cost that the paper does not offset (future work is to improve this efficiency). Also observed: it is not lossless everywhere (1/19 cases in Table 1 is not state-of-the-art, and only 20/35 cross-architecture cells are lossless); the curriculum-learning difficulty measure is specific to homophily-based node classification and does not obviously transfer to graph-level classification; GCond itself runs out of memory at ogbn-arxiv r=5%, so GEOM's \"lossless\" claim there has no GCond comparison point at that ratio." } }, { "id": "a-survey-on-graph-2024", "type": "paper", "title": "A Survey on Graph Condensation", "arxiv": "2402.02000", "url": null, "date": "2024-02-03", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Hongjia Xu", "Liangliang Zhang", "Yao Ma", "Sheng Zhou", "Zhuonan Zheng", "Bu Jiajun" ], "kind": "survey", "settings": [ "graph" ], "families": [], "tier": "catalogued", "links": { "code": "https://github.com/Frostland12138/Awesome-Graph-Condensation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Surveys graph condensation, giving a formal problem definition and a taxonomy that splits methods by optimization objective into three types and by output into graph-modifying versus fully synthetic approaches, and reviews the datasets and evaluation metrics used in the field.", "abstract": "Analytics on large-scale graphs have posed significant challenges to computational efficiency and resource requirements. Recently, Graph condensation (GC) has emerged as a solution to address challenges arising from the escalating volume of graph data. The motivation of GC is to reduce the scale of large graphs to smaller ones while preserving essential information for downstream tasks. For a better understanding of GC and to distinguish it from other related topics, we present a formal definition of GC and establish a taxonomy that systematically categorizes existing methods into three types based on its objective, and classify the formulations to generate the condensed graphs into two categories as modifying the original graphs or synthetic completely new ones. Moreover, our survey includes a comprehensive analysis of datasets and evaluation metrics in this field. Finally, we conclude by addressing challenges and limitations, outlining future directions, and offering concise guidelines to inspire future research in this field.", "awesome_sections": [ "Applications / Graph Neural Network / Survey" ], "notes": "" }, { "id": "dataset-condensation-driven-machine-2024", "type": "paper", "title": "Dataset Condensation Driven Machine Unlearning", "arxiv": "2402.00195", "url": null, "date": "2024-01-31", "venue": { "name": "arXiv 2024", "status": "preprint" }, "authors": [ "Junaid Iqbal Khan" ], "kind": "application", "settings": [ "image-classification" ], "families": [], "applications": [ "unlearning", "privacy" ], "tier": "catalogued", "links": { "code": "https://github.com/algebraicdianuj/DC_U", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Proposes new dataset condensation techniques used as a building block of an unlearning scheme, condensing training data so that an approximate-unlearning update is cheaper to compute while balancing privacy, utility and efficiency; also proposes using the resulting \"condensed model\" to retrain arbitrary models unaffected by the forgotten samples and applies the scheme to defend against membership inference and model inversion attacks.", "abstract": "The current trend in data regulation requirements and privacy-preserving machine learning has emphasized the importance of machine unlearning. The naive approach to unlearning training data by retraining over the complement of the forget samples is susceptible to computational challenges. These challenges have been effectively addressed through a collection of techniques falling under the umbrella of machine unlearning. However, there still exists a lack of sufficiency in handling persistent computational challenges in harmony with the utility and privacy of unlearned model. We attribute this to the lack of work on improving the computational complexity of approximate unlearning from the perspective of the training dataset. In this paper, we aim to fill this gap by introducing dataset condensation as an essential component of machine unlearning in the context of image classification. To achieve this goal, we propose new dataset condensation techniques and an innovative unlearning scheme that strikes a balance between machine unlearning privacy, utility, and efficiency. Furthermore, we present a novel and effective approach to instrumenting machine unlearning and propose its application in defending against membership inference and model inversion attacks. Additionally, we explore a new application of our approach, which involves removing data from `condensed model', which can be employed to quickly train any arbitrary model without being influenced by unlearning samples. The corresponding code is available at \\href{https://github.com/algebraicdianuj/DC_U}{URL}.", "awesome_sections": [ "Applications / Machine Unlearning" ], "notes": "families left empty: abstract does not name the matching mechanism behind the new condensation techniques." }, { "id": "survey-graph-reduction-2024", "type": "paper", "title": "A Comprehensive Survey on Graph Reduction: Sparsification, Coarsening, and Condensation", "short": "Graph reduction survey", "arxiv": "2402.03358", "url": null, "date": "2024-01-29", "venue": { "name": "IJCAI 2024", "status": "conference" }, "authors": [ "Mohammad Hashemi", "Shengbo Gong", "Juntong Ni", "Wenqi Fan", "B. Aditya Prakash", "Wei Jin" ], "kind": "survey", "settings": [ "graph" ], "families": [], "tier": "catalogued", "links": { "code": "https://github.com/Emory-Melody/awesome-graph-reduction", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "This survey unifies graph sparsification, coarsening and condensation under one definition, introduces a hierarchical taxonomy of the challenges each addresses, reviews the technical details and practical applications of methods in all three families, and lays out research directions for keeping graph reduction effective as graphs continue to grow.", "abstract": "Many real-world datasets can be naturally represented as graphs, spanning a wide range of domains. However, the increasing complexity and size of graph datasets present significant challenges for analysis and computation. In response, graph reduction, or graph summarization, has gained prominence for simplifying large graphs while preserving essential properties. In this survey, we aim to provide a comprehensive understanding of graph reduction methods, including graph sparsification, graph coarsening, and graph condensation. Specifically, we establish a unified definition for these methods and introduce a hierarchical taxonomy to categorize the challenges they address. Our survey then systematically reviews the technical details of these methods and emphasizes their practical applications across diverse scenarios. Furthermore, we outline critical research directions to ensure the continued effectiveness of graph reduction techniques, as well as provide a comprehensive paper list at \\url{https://github.com/Emory-Melody/awesome-graph-reduction}. We hope this survey will bridge literature gaps and propel the advancement of this promising field.", "notes": "Setting corrected from the draft's image-classification to graph: this survey covers graph sparsification/coarsening/condensation exclusively." }, { "id": "importance-aware-adaptive-dataset-2024", "type": "paper", "title": "Importance-Aware Adaptive Dataset Distillation", "short": "IADD", "arxiv": "2401.15863", "url": null, "date": "2024-01-29", "venue": { "name": "NN 2024", "status": "journal" }, "authors": [ "Guang Li", "Ren Togo", "Takahiro Ogawa", "Miki Haseyama" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "gradient-matching", "optimization-recipes" ], "applications": [ "medical" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Observes that parameter-/gradient-matching distillation treats every network parameter as equally important and proposes IADD, which automatically assigns importance weights to different parameters during distillation to synthesize more robust distilled sets. Reports outperforming other parameter-matching SOTA methods on multiple benchmarks and in cross-architecture generalization, and validates the method on real-world COVID-19 detection.", "abstract": "Herein, we propose a novel dataset distillation method for constructing small informative datasets that preserve the information of the large original datasets. The development of deep learning models is enabled by the availability of large-scale datasets. Despite unprecedented success, large-scale datasets considerably increase the storage and transmission costs, resulting in a cumbersome model training process. Moreover, using raw data for training raises privacy and copyright concerns. To address these issues, a new task named dataset distillation has been introduced, aiming to synthesize a compact dataset that retains the essential information from the large original dataset. State-of-the-art (SOTA) dataset distillation methods have been proposed by matching gradients or network parameters obtained during training on real and synthetic datasets. The contribution of different network parameters to the distillation process varies, and uniformly treating them leads to degraded distillation performance. Based on this observation, we propose an importance-aware adaptive dataset distillation (IADD) method that can improve distillation performance by automatically assigning importance weights to different network parameters during distillation, thereby synthesizing more robust distilled datasets. IADD demonstrates superior performance over other SOTA dataset distillation methods based on parameter matching on multiple benchmark datasets and outperforms them in terms of cross-architecture generalization. In addition, the analysis of self-adaptive weights demonstrates the effectiveness of IADD. Furthermore, the effectiveness of IADD is validated in a real-world medical application such as COVID-19 detection.", "awesome_sections": [ "Applications / Medical" ], "notes": "" }, { "id": "survey-graph-condensation-2025", "type": "paper", "title": "Graph Condensation: A Survey", "short": "Graph condensation survey", "arxiv": "2401.11720", "url": null, "date": "2024-01-22", "venue": { "name": "TKDE 2025", "status": "journal" }, "authors": [ "Xinyi Gao", "Junliang Yu", "Tong Chen", "Guanhua Ye", "Wentao Zhang", "Hongzhi Yin" ], "kind": "survey", "settings": [ "graph" ], "families": [], "tier": "catalogued", "links": { "code": "https://github.com/xygaog/graph-condensation-papers", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "This survey organizes graph condensation research into five categories aligned with evaluation criteria (effectiveness, generalization, efficiency, fairness, robustness), examines optimization strategies and condensed-graph generation as the two core components across methods, empirically compares representative methods against the five criteria, and surveys applications and open-source libraries in the field.", "abstract": "The rapid growth of graph data poses significant challenges in storage, transmission, and particularly the training of graph neural networks (GNNs). To address these challenges, graph condensation (GC) has emerged as an innovative solution. GC focuses on synthesizing a compact yet highly representative graph, enabling GNNs trained on it to achieve performance comparable to those trained on the original large graph. The notable efficacy of GC and its broad prospects have garnered significant attention and spurred extensive research. This survey paper provides an up-to-date and systematic overview of GC, organizing existing research into five categories aligned with critical GC evaluation criteria: effectiveness, generalization, efficiency, fairness, and robustness. To facilitate an in-depth and comprehensive understanding of GC, this paper examines various methods under each category and thoroughly discusses two essential components within GC: optimization strategies and condensed graph generation. We also empirically compare and analyze representative GC methods with diverse optimization strategies based on the five proposed GC evaluation criteria. Finally, we explore the applications of GC in various fields, outline the related open-source libraries, and highlight the present challenges and novel insights, with the aim of promoting advancements in future research. The related resources can be found at https://github.com/XYGaoG/Graph-Condensation-Papers.", "notes": "Setting corrected from the draft's image-classification to graph: this survey is entirely about graph condensation, and the graph setting file already lists it as a key paper." }, { "id": "medsynth-2024", "type": "paper", "title": "MedSynth: Leveraging Generative Model for Healthcare Data Sharing", "short": "MedSynth", "arxiv": null, "url": "https://link.springer.com/chapter/10.1007/978-3-031-72390-2_61", "date": "2024-01-01", "venue": { "name": "MICCAI 2024", "status": "conference" }, "authors": [ "Renuga Kanagavelu" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "generator-as-dataset" ], "applications": [ "medical", "privacy" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "abstract" }, "summary": "Condenses the knowledge in large medical datasets into a small generative model, combining an attention-based generator with a vision transformer so the model can produce a compact set of representative synthetic medical images; sharing the generator across hospitals avoids disclosing raw patient data. Reports outperforming state-of-the-art comparison methods and successfully defending against state-of-the-art membership-inference attacks.", "abstract": null, "awesome_sections": [ "Applications / Medical" ], "notes": "Full author list not confirmed beyond the first author (Kanagavelu et al.)." }, { "id": "differentially-private-dataset-condensation-2024", "type": "paper", "title": "Differentially Private Dataset Condensation", "arxiv": null, "url": "https://www.ndss-symposium.org/ndss-paper/auto-draft-542/", "date": "2024-01-01", "venue": { "name": "NDSS 2024 Workshop", "status": "workshop" }, "authors": [ "Tianhang Zheng", "Baochun Li" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "gradient-matching", "distribution-matching" ], "applications": [ "privacy" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "abstract" }, "summary": "Proposes two differentially-private dataset condensation algorithms: LDPDC, a linear DC method that runs on a low-end CPU, and NDPDC, which uses a neural network for representation extraction under a DP feature/gradient-matching objective. Reports LDPDC performs comparably to recent privacy-preserving generative methods, while NDPDC gives acceptable DP guarantees with only mild utility loss relative to plain distribution matching.", "abstract": null, "awesome_sections": [ "Applications / Privacy" ], "notes": "" }, { "id": "textual-dataset-distillation-via-2024", "type": "paper", "title": "Textual Dataset Distillation via Language Model Embedding", "arxiv": null, "url": "https://aclanthology.org/2024.findings-emnlp.733/", "date": "2024-01", "venue": { "name": "EMNLP 2024", "status": "conference" }, "authors": [ "Yefan Tao", "Luyang Kong", "Andrey Kan", "Laurent Callot" ], "kind": "method", "settings": [ "text" ], "families": [], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "aclanthology", "depth": "abstract" }, "summary": "This method distills text classification datasets into language-model embeddings rather than into embeddings tied to one specific model, aiming for a model-agnostic, data-efficient distilled representation with better transferability than prior embedding-space text distillation. On IMDB and AG-News the paper reports results comparable to or better than parameter-efficient fine-tuning (LoRA) and than prior model-dependent text distillation methods, with faster processing.", "abstract": null, "notes": "Non-arXiv (ACL Anthology); full author list and abstract content read from the landing page (abstract left null per policy). The exact matching mechanism (which of gradient/distribution matching underlies the embedding-space distillation) is not clear from the available text, so families is left empty." }, { "id": "information-compensation-a-fix-2024", "type": "paper", "title": "Information Compensation: A Fix for Any-scale Dataset Distillation", "arxiv": null, "url": "https://openreview.net/forum?id=2SnmKd1JK4", "date": "2024-01", "venue": { "name": "ICLR 2024 Workshop", "status": "workshop" }, "authors": [ "Peng Sun" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "metadata" }, "summary": "Addresses information loss that arises when decoupled ('squeeze-recover-relabel' style) dataset distillation is applied across different compression scales or budgets. Method details could not be independently verified: OpenReview returned only a browser-verification challenge, and no alternative fetchable copy of the abstract was found.", "abstract": null, "awesome_sections": [ "Main / Decoupled Distillation" ], "notes": "Abstract not accessible (OpenReview verification blocked on both the API and forum page); title and ICLR 2024 Workshop (Data-centric Machine Learning Research) venue confirmed via web search and citations in later papers by the same author (RDED, CIM). Flag for a full-text pass when accessible." }, { "id": "gsdd-2024", "type": "paper", "title": "GSDD: Generative Space Dataset Distillation for Image Super-resolution", "short": "GSDD", "arxiv": null, "url": "https://ojs.aaai.org/index.php/AAAI/article/view/28534", "date": "2024-01", "venue": { "name": "AAAI 2024", "status": "conference" }, "authors": [ "Haiyu Zhang", "Shaolin Su", "Yu Zhu", "Jinqiu Sun", "Yanning Zhang" ], "kind": "method", "settings": [ "dense-prediction" ], "families": [ "generative-latent-optimization" ], "facets": { "prior": [ "gan" ] }, "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "abstract" }, "summary": "GSDD distills a low-resolution/high-resolution image-pair dataset for super-resolution by optimizing codes in the latent space of a pretrained GAN via GAN-inversion, rather than selecting or synthesizing pixels directly, so the stored artifact is a small set of latent codes plus the frozen generator. The paper reports super-resolution performance comparable to prior state-of-the-art distillation methods with about an 8x increase in training efficiency and roughly 93.2% less storage, and shows generalization to real-world degraded images.", "abstract": null, "notes": "Non-arXiv (AAAI OJS); author list and abstract content read from the landing page, abstract left null per policy." }, { "id": "dcod-2024", "type": "paper", "title": "Fetch and Forge: Efficient Dataset Condensation for Object Detection", "short": "DCOD", "arxiv": null, "url": "https://openreview.net/forum?id=m8MElyzuwp", "date": "2024-01", "venue": { "name": "NeurIPS 2024", "status": "conference" }, "authors": [ "Ding Qi", "Jian Li", "Jinlong Peng", "Bo Zhao", "Shuguang Dou", "Jialin Li", "Jiangning Zhang", "Yabiao Wang", "Chengjie Wang", "Cairong Zhao" ], "kind": "method", "settings": [ "dense-prediction" ], "families": [ "decoupled-synthesis" ], "facets": { "prior": [ "trained-experts" ], "parameterization": [ "pixels" ] }, "builds_on": [ "sre2l-2023", "dim-2023", "dc-2021", "mtt-2022" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "full-text" }, "summary": "Proposes DCOD, the first dataset-condensation framework for object detection, in a two-stage Fetch-and-Forge pipeline: localization and classification information from the real dataset is first embedded into model parameters (Fetch), then synthetic images are reconstructed by inverting that model (Forge); a Foreground-Background Decoupling strategy and an Incremental PatchExpand step increase multi-instance diversity within a fixed storage budget. At a 1% compression rate it reports 46.4% and 24.7% AP50 on VOC and COCO respectively.", "abstract": null, "awesome_sections": [ "Applications / Object Detection" ], "notes": "Full text obtained via the NeurIPS 2024 proceedings PDF (OpenReview PDF was bot-blocked; no arXiv version exists). One internal inconsistency in the source paper: the abstract and Table 2 both report 24.7% AP50 on COCO at 1% compression, but the page-3 bullet list says 24.5% — the atlas follows the abstract/Table 2 value (24.7%).", "explained": { "before": "Classification-focused dataset condensation splits into meta-learning (bi-level optimization, e.g. DD, KIP, RFAD, FRePo) and data-matching (gradient matching DC/DSA, distribution matching DM/CAFE/DataDAM/ IDM, trajectory matching MTT) frameworks, all of which involve a bi-level or matching loop tied to a single object/label per image; decoupled-synthesis methods for classification (SRe2L) instead train one model once on real data and invert it to synthesize images via BN-statistics matching, and DiM compresses a dataset into a generator's parameters rather than the images themselves.", "problem": "Object detection has two properties that break both frameworks: (i) it is inherently multitask (simultaneous localization and classification of possibly many objects per image), which the bi-level/ matching machinery was not designed to handle, and (ii) detection datasets are large-scale and high-resolution, which makes the existing bi-level optimization loop's cost (already limited to small datasets like CIFAR and simple backbones like ConvNet-3/AlexNet/ResNet-18 in classification) impractical to extend to complex detectors and high-resolution multi-object images.", "idea": "Decouple detection dataset condensation into two separate, non-bi-level stages inspired by SRe2L-style model inversion: first train a real detector once on the full dataset (Fetch — key localization/ classification information is stored implicitly in the detector's weights), then freeze that detector and synthesize images by inverting it (Forge), using two detection-specific mechanisms — Foreground- Background Decoupling and Incremental PatchExpand — to handle the fact that a detection image, unlike a classification image, contains multiple objects at different positions, sizes, and categories.", "method": "Stage I (Fetch): train a detector $\\psi_\\theta$ (YOLOv3-SPP) on the real dataset $\\mathcal{T}$ with the standard composite detection loss $\\mathcal{L}_{det}=\\mathcal{L}_{loc}+\\mathcal{L}_{cls}$ (Eq. 3-4); this is a single standard training run, not a bi-level loop. Stage II (Forge): freeze the trained detector; initialize synthetic images by random sampling from the real dataset with their real bounding-box/class targets; apply Foreground-Background Decoupling ($\\mathcal{F}_{BD}$) — a binary mask from the box coordinates separates foreground/background, background pixels are suppressed by a factor $\\alpha<1$ (limiting how much the background updates, to preserve context and avoid blending unrelated scene semantics) while foreground pixels are enhanced via random erasure to force the model to keep refining them; apply Incremental PatchExpand ($\\mathcal{I}_{PE}$) — pad each image to a uniform size, divide it into a $k\\times k$ grid of patches, and optimize each patch toward a different target label, with the number of active patches increased incrementally (curriculum-style) during optimization to add diversity of object position/size/shape. Optimize the synthetic images $\\widetilde{X}$ to minimize $L_{det}(\\psi_{\\theta_\\mathcal{T}}(\\widetilde{X}),\\widetilde{A}) + R_{reg}$, where $R_{reg}=R_{pixel}+ R_{feature}$: $R_{pixel}$ is a total-variation + $\\ell_2$ term (Eq. 9-10), $R_{feature}$ matches per-layer batch-norm mean/variance statistics of the synthetic images to the frozen detector's stored BN statistics (Eq. 11, an SRe2L-style term). No labels are learned; bounding boxes and classes are fixed from real annotations throughout.", "evidence": "Table 1 (Pascal VOC, YOLOv3-SPP condensation+evaluation network, hard labels, compression ratios 0.5%/1%/2%): DCOD reaches mAP 23.7%, AP50 50.7% at ratio 2% (full dataset 46.5% mAP, 76.4% AP50), vs. best coreset baseline (Herding) 8.5% mAP/28.1% AP50 at the same ratio — roughly 3x the coreset mAP. Table 2 (MS COCO, same setup, ratios 0.25%/0.5%/1%): at 1%, DCOD reaches 12.1% mAP, 24.7% AP50 (full 36.1% mAP, 63.6% AP50) vs. best coreset baseline (Herding) 4.1% mAP/12.5% AP50. Table 3 (VOC, 1% ratio, ablation): removing both $\\mathcal{F}_{BD}$ and $\\mathcal{I}_{PE}$ (baseline = plain model-inversion Forge) gives only 8.3% mAP/25.9% AP50; adding $\\mathcal{I}_{PE}$ alone raises this to 17.7% mAP; adding $\\mathcal{F}_{BD}$ alone gives 10.0% mAP; combined they reach 19.8% mAP/46.4% AP50 — an 11.5-point mAP improvement over the inversion-only baseline, and the paper notes that without these two components \"the detection loss quickly converges in early iterations, missing key positional and classification details.\" Table 4 (cross-architecture, YOLOv3-SPP-condensed data evaluated with Faster R-CNN, a two-stage detector): performance drops substantially at low compression (VOC 0.5%: 14.2% mAP same-arch vs. 6.3% cross-arch), narrowing but not closing at 1% (19.8% vs. 13.8% mAP). Table 5 (initialization ablation, VOC 0.5%): Random initialization (14.2% mAP) beats K-center (11.9%) and Herding (11.5%), the opposite finding from image classification, attributed to core-set methods not accounting for detection-specific class/object-size distribution. Figure 7 shows DCOD's advantage over random selection shrinks and the two curves converge once the compression ratio exceeds ~20%.", "limitations": "Explicitly not extended to more complex/transformer-based detectors like DETR, which the authors state \"might require more specialized designs to accommodate their structures\"; cross-architecture transfer (YOLO-condensed data evaluated with Faster R-CNN) remains substantially weaker than same-architecture evaluation, especially at low compression ratios, and the paper states this \"remains an area that needs improvement\"; all experiments use a single detector family (YOLOv3-SPP) for condensation; labels/boxes are fixed from real annotations rather than learned or relabeled, so the method inherits whatever localization noise exists in the original dataset; only a single GPU (V100) wall-clock/memory budget is implied by the experimental setup but not explicitly quantified as a cost table." } }, { "id": "d2m-2024", "type": "paper", "title": "Data-to-Model Distillation: Data-Efficient Learning Framework", "short": "D2M", "arxiv": null, "url": "https://www.ecva.net/papers/eccv_2024/papers_ECCV/html/6020_ECCV_2024_paper.php", "date": "2024-01", "venue": { "name": "ECCV 2024", "status": "conference" }, "authors": [ "Ahmad Sajedi", "Samir Khaki", "Lucy Z. Liu", "Ehsan Amjadian", "Yuri A. Lawryshyn", "Konstantinos N. Plataniotis" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "generator-as-dataset" ], "facets": { "parameterization": [ "generator-weights" ], "labels": "hard", "prior": [ "gan" ], "regime": "generator-fine-tuning", "scale": [ "imagenet-1k" ] }, "applications": [ "nas" ], "tier": "catalogued", "links": { "code": "https://github.com/DataDistillation/D2M", "project": "https://datadistillation.github.io/D2M/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "full-text" }, "summary": "Transfers knowledge into the parameters of a pretrained generative model by matching real and synthetic representations, rather than storing pixels, so the same trained generator serves any distillation ratio without retraining; scales to ImageNet-1K at 128x128, validated on 15 datasets with an application to neural architecture search.", "abstract": null, "awesome_sections": [ "Main / Generative Distillation / GAN" ], "notes": "Reclassified from generative-latent-optimization to generator-as-dataset per docs/taxonomy.md's explicit listing of D2M as a generator-as-dataset example (the method fine-tunes/trains the generator's parameters as the stored artifact).", "explained": { "before": "DC, DSA, DM, CAFE, DAM, ATOM, IDC, KIP, FRePo, MTT, FTD, TESLA all distill into pixels and must fully re-distill when IPC changes; DiM already stores the dataset in a generator's weights via a models-pool logits-matching loss, but only demonstrated CIFAR/MNIST-scale results; GLaD and IT-GAN optimize a frozen generator's latents rather than its weights, so their storage still grows linearly with IPC (one latent per image) and both need complete retraining whenever the distillation ratio changes.", "problem": "Pixel-parameterized distillation methods scale their storage and re-distillation cost linearly with IPC and resolution, degrade on unseen architectures (ResNet, DenseNet, ViT) because they are tuned to a small ConvNet, and had not been scaled past 64x64 ImageNet-1K or 128x128 subsets before this paper, largely because pixel-space optimization at that resolution is memory- and compute-prohibitive.", "method": "A BigGAN pretrained on the target dataset (StyleGAN-XL, a conditional GAN, and a CVAE are also tested as alternative generators in an ablation) is fine-tuned with two losses computed via a randomly-sampled network $\\phi_\\theta$ from a model pool of Depth-$n$ ConvNets and ResNet-18/32 with varied seeds: an embedding-matching loss $\\mathcal{L}_{\\text{EM}}$ that matches channel-wise attention maps (from intermediate layers) and final-layer features between real and generator-sampled batches sharing labels $Y$, and a prediction-matching loss $\\mathcal{L}_{\\text{PM}}$ that minimizes the KL divergence between temperature-softened ($T$) softmax predictions on real vs. generated logits (a KD-style soft-label matching signal used only during distillation, not carried to evaluation). The generator is updated by SGD on $\\mathcal{L}=\\mathcal{L}_{\\text{EM}}+\\lambda\\mathcal{L}_{\\text{PM}}$ (batch size 128, $K{=}60$ epochs, default $\\lambda{=}10$ in the sensitivity ablation, $\\lambda{=}100$/$T{=}4$ in the main implementation-detail setup, on two RTX A6000 GPUs). At evaluation, $\\mathcal{S}=G(Z)=\\{(s_i,y_i)\\}$ is sampled from the frozen fine-tuned generator with hard labels $y_i$ equal to the conditioning class, and a fresh network is trained on these images from scratch — the same generator serves any IPC without re-optimization.", "evidence": "CIFAR-10/100 and Tiny-ImageNet, ConvNet eval, hard labels (Table 1): CIFAR-10 IPC1 D2M 50.2% vs. FRePo 46.8%/MTT 46.3%; IPC10 67.8% vs. FRePo 65.5%; IPC50 74.4% vs. FRePo 71.7% (full-dataset upper bound 84.8%); CIFAR-100 IPC1 29.8% vs. FRePo 28.7%. ImageNet-1K at 64x64 (Table 2, hard labels): IPC1 D2M 8.3% vs. TESLA 7.7%/FRePo 7.5%; IPC50 D2M 28.3% vs. DAM 15.5% (a 19.1-point gap the paper highlights via Fig. 3, full-dataset 33.8%). 128x128 ImageNet-1K subsets (Table 2): ImageNette IPC1 D2M 49.9% vs. GLaD 38.7%; ImageWoof IPC1 32.3% vs. GLaD 23.4%; ImageSquawk IPC1 40.3% vs. GLaD 35.8%. Head-to-head with GLaD plugged into DC/DM/MTT on ImageFruit/ImageMeow/ImageNet-A-E at IPC1 (Table 3, all re-run by the authors): D2M beats every DC+GLaD/DM+GLaD/MTT+GLaD variant, e.g. ImageNet-A 55.2% vs. MTT+GLaD 50.7%. Re-distillation cost (Table 4): total GPU-hours to cover IPC1/10/50 on CIFAR-10 is 4.9 for D2M (all spent once at IPC1; redeploying to IPC10/50 costs 0 additional GPU-hours) vs. 47.4 for MTT and 10.3 for DC. Cross-architecture (Table 5, CIFAR-10 IPC50, unseen AlexNet/VGG-11/ResNet-50/DenseNet-121/ViT): D2M average 63.6% vs. best baseline DAM 59.7% (+3.9 points). Key ablation (loss components, CIFAR-10 IPC10): removing $\\mathcal{L}_{\\text{PM}}$ drops accuracy by 7.9 points and removing $\\mathcal{L}_{\\text{EM}}$ by 10.3 points relative to the full loss, whose combination gives a 12.7-point gain over a distillation-loss-free generator baseline — both matching terms are needed, with embedding matching contributing more. Generator-choice ablation (Table 7, CIFAR-10): StyleGAN-XL slightly outperforms BigGAN (75.2% vs. 74.4% at IPC50) but BigGAN is kept as default for its compute/performance balance — all four generators tested are trained on the target dataset, so this does not test an off-target generator. NAS application (Table 8): D2M-selected architectures reach 88.9% correlation-weighted accuracy and 0.80 Spearman rank correlation with real-data ranking, best among Random/DSA/DM/DAM/CAFE proxy sets.", "limitations": "Every generator tested (BigGAN, CGAN, StyleGAN-XL, CVAE) is trained or fine-tuned on the exact target dataset, so, like DiM, none of the reported gain isolates a generator not matched to the target distribution; the paper's own stated limitation is that D2M needs more GPU memory than pixel methods at IPC1 (the fixed cost of the generator dominates at the smallest budget); evaluation is hard-label only despite using soft KL-divergence matching internally; the main experiments report GPU-hours for re-distillation but not the upfront cost of pretraining the base generator itself; ImageNet-1K experiments stop at 64x64 (only the 128x128 subsets, not full ImageNet-1K, are run at higher resolution)." } }, { "id": "avdd-2024", "type": "paper", "title": "Audio-Visual Dataset Distillation", "short": "AVDD", "arxiv": null, "url": "https://openreview.net/forum?id=IJlbuSrXmk", "date": "2024-01", "venue": { "name": "TMLR 2024", "status": "journal" }, "authors": [ "Saksham Singh Kushwaha", "Siva Sai Nagender Vasireddy", "Kai Wang", "Yapeng Tian" ], "kind": "method", "settings": [ "audio-visual-omni" ], "families": [ "distribution-matching" ], "facets": { "prior": [ "random-networks" ], "labels": "hard", "regime": "single-level", "parameterization": [ "pixels" ], "scale": [ "large-scale-other" ] }, "tier": "core", "links": { "code": "https://github.com/sakshamsingh1/AVDD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "openreview", "depth": "full-text" }, "summary": "Extends distribution matching to audio-visual data, first matching visual-only and audio-only feature distributions with randomly initialized networks separately, then adding a joint matching loss that implicitly cross-matches real audio against synthetic visual features (and vice versa) and a modality-gap matching loss that aligns the audio-visual gap between real and synthetic data, plus herding-based initialization; on VGGS-10K at 10 images-per-class it reports 54.99% recognition accuracy versus 43.85% for plain distribution matching and 68.24% for the full dataset (Table 3), and raises audio-to-visual retrieval Recall@1 from 8.66% (DM) to 19.33% (Table 5).", "abstract": null, "builds_on": [ "dm-2023" ], "awesome_sections": [ "Main / Multimodal Distillation" ], "explained": { "before": "Distribution matching (DM) minimizes the feature-distance between real and synthetic data using randomly initialized networks, applied so far only within a single modality (image classification). MTT (trajectory matching) is used as a second baseline. No prior method addressed paired audio-visual data.", "problem": "A naive extension of DM to audio-visual data trains a visual-only DM loss and an audio-only DM loss independently and sums them ($\\mathcal L^{av}_{base}=\\mathcal L^a_{base}+\\mathcal L^v_{base}$). The paper diagnoses that this cannot capture cross-modal correspondence: because the visual network $\\psi_{\\theta_v}$ and audio network $\\psi_{\\theta_a}$ are separate, randomly initialized networks with no shared structure, matching each modality's distribution separately creates a \"modality gap\" - there is nothing in the objective that ties a synthetic audio sample to its paired synthetic visual sample the way real audio-visual pairs are tied together.", "idea": "Add two loss terms on top of per-modality DM that explicitly couple the two modalities: a joint matching loss that matches the *sum* of real audio and visual features against the sum of synthetic audio and visual features (which, when expanded, implicitly cross-matches real audio against synthetic visual features and vice versa), and a modality-gap matching loss that aligns the *difference* between audio and visual features so the audio-visual gap itself is consistent between real and synthetic data - both computed via MMD-style feature-mean matching using randomly initialized visual/audio ConvNets.", "method": "Synthetic video frames ($224\\times224$) and audio spectrograms are directly optimized pixels, one set per class. A randomly initialized visual ConvNet $\\psi_{\\theta_v}$ and randomly initialized audio ConvNet $\\psi_{\\theta_a}$ (fresh random weights sampled per iteration, as in DM; not trained across iterations) extract features under differentiable Siamese augmentation $\\mathcal A_\\omega(\\cdot)$. The base loss (Eq. 1) sums per-modality DM losses. Joint matching (Eq. 2) matches $\\mathcal D^r=\\bar R^a+\\bar R^v$ against $\\mathcal D^s=\\bar S^a+\\bar S^v$ (sums of mean real/ synthetic audio and visual features), which algebraically bounds $\\|(\\bar R^a-\\bar S^v)\\|^2+\\|(\\bar R^v-\\bar S^a)\\|^2$ - an implicit cross-modal match. Modality-gap matching (Eq. 3) instead matches $\\mathcal D^{av}=\\bar R^a+\\bar S^v$ against $\\mathcal D^{va}=\\bar R^v+\\bar S^a$, which reduces to matching the real audio-visual gap $(\\bar R^a-\\bar R^v)$ against the synthetic gap $(\\bar S^a-\\bar S^v)$. The final loss (Eq. 4) sums base + weighted JM + weighted MGM. Synthetic data is initialized via herding (coreset selection) rather than random real samples, and a \"factor\" technique increases the number of extracted features per stored sample at no extra storage cost. At evaluation, fresh visual and audio ConvNets are trained from scratch on the synthetic set for 30 epochs (5000 iterations/run, 3 runs), fused by late-fusion ensembling (shown to beat concatenation, summation, and attention fusion).", "evidence": "On VGGS-10K (10-class VGGSound subset) at IPC=10 (Table 3): audio-visual ensemble accuracy 54.99$\\pm$1.73% versus DM 43.85$\\pm$1.75%, MTT 36.79$\\pm$1.97%, herding 39.89$\\pm$1.64%, random 32.01$\\pm$1.64%, and whole-data upper bound 68.24$\\pm$0.75%. At IPC=1: 40.41$\\pm$1.81% vs DM 36.54$\\pm$2.52%; at IPC=20: 58.04$\\pm$1.68% vs DM 49.01$\\pm$2.44%. On AVE at IPC=10: 36.82$\\pm$0.88% vs DM 28.14$\\pm$1.80%, whole-data 52.20$\\pm$0.38%. Audio-visual retrieval (Table 5, IPC=20, class-wise retrieval on 5-samples-per-class test subsets): A$\\to$V R@1 19.33$\\pm$2.30% vs DM 8.66$\\pm$1.15% vs whole-data 44.00$\\pm$2.00%; V$\\to$A R@1 27.33$\\pm$2.30% vs DM 11.33$\\pm$3.05% vs whole-data 45.33$\\pm$5.03%. The key ablation (Table 4, IPC=10, additive from a base DM run) shows: random-init 32.01% (VGGS-10K) / 20.00% (AVE); switching to herding init 39.89%/26.86%; adding the factor technique 40.28%/31.80%; adding the base audio-visual DM loss 45.31%/34.80%; adding joint matching (JM) 49.07%/35.13%; adding modality-gap matching (MGM) on top reaches the full 54.99%/ 36.82% - every added component improves both datasets, with MGM and JM together contributing the largest joint increment over the herding+factor+base configuration. No GPU-hour/wall-clock cost is reported in this (4-page workshop) version.", "limitations": "This full-text read is the 4-page SightSound 2024 workshop precursor rather than the TMLR 2024 journal version; the journal abstract indicates MUSIC-21 and fuller VGGSound evaluations were added later and are not covered by this reading. Retrieval evaluation is class-wise (5 samples per class) rather than instance-level, since the paper states its DM-based objective targets semantic, not instance-level, alignment. AVE at IPC=20 could not scale up for the MTT baseline (\"-\" in Table 3), limiting the baseline comparison at that budget. Audio and visual data are distilled and evaluated as one-second clips with a single center frame plus a spectrogram, not full video; both modality networks are randomly initialized at each iteration rather than pretrained, so the method's dependence on that specific (untrained-network) DM recipe is untested against a pretrained-encoder alternative." }, "notes": "Full text read is the 4-page SightSound 2024 workshop precursor (same authors and method); the TMLR 2024 journal version adds MUSIC-21 and full VGGSound evaluations per its abstract." }, { "id": "mim4dd-2023", "type": "paper", "title": "MIM4DD: Mutual Information Maximization for Dataset Distillation", "short": "MIM4DD", "arxiv": "2312.16627", "url": null, "date": "2023-12-27", "venue": { "name": "NeurIPS 2023", "status": "conference" }, "authors": [ "Yuzhang Shang", "Zhihang Yuan", "Yan Yan" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Adds a mutual-information objective, estimated via an InfoNCE-style contrastive loss between same-label (positive) and different-label (negative) real/synthetic sample pairs, as a plug-in regularizer for existing gradient- and trajectory-matching distillation methods, on the argument that maximizing shared information is a more principled compression criterion than the heuristic indicators these methods already match.", "abstract": "Dataset distillation (DD) aims to synthesize a small dataset whose test performance is comparable to a full dataset using the same model. State-of-the-art (SoTA) methods optimize synthetic datasets primarily by matching heuristic indicators extracted from two networks: one from real data and one from synthetic data (see Fig.1, Left), such as gradients and training trajectories. DD is essentially a compression problem that emphasizes maximizing the preservation of information contained in the data. We argue that well-defined metrics which measure the amount of shared information between variables in information theory are necessary for success measurement but are never considered by previous works. Thus, we introduce mutual information (MI) as the metric to quantify the shared information between the synthetic and the real datasets, and devise MIM4DD numerically maximizing the MI via a newly designed optimizable objective within a contrastive learning framework to update the synthetic dataset. Specifically, we designate the samples in different datasets that share the same labels as positive pairs and vice versa negative pairs. Then we respectively pull and push those samples in positive and negative pairs into contrastive space via minimizing NCE loss. As a result, the targeted MI can be transformed into a lower bound represented by feature maps of samples, which is numerically feasible. Experiment results show that MIM4DD can be implemented as an add-on module to existing SoTA DD methods.", "awesome_sections": [ "Main / Better Optimization" ] }, { "id": "m3d-2024", "type": "paper", "title": "M3D: Dataset Condensation by Minimizing Maximum Mean Discrepancy", "short": "M3D", "arxiv": "2312.15927", "url": null, "date": "2023-12-26", "venue": { "name": "AAAI 2024", "status": "conference" }, "authors": [ "Hansong Zhang", "Shikun Li", "Pengju Wang", "Dan Zeng", "Shiming Ge" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "facets": { "parameterization": [ "pixels", "downsampled-multiformation" ], "labels": "hard", "prior": [ "random-networks" ], "regime": "single-level", "scale": [ "mnist-cifar", "imagenet-subsets" ] }, "tier": "notable", "links": { "code": "https://github.com/Hansong-Zhang/M3D", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Argues that prior distribution-matching methods only align first-moment (mean) statistics and so under-match the real and synthetic feature distributions, and proposes minimizing the maximum mean discrepancy in a reproducing kernel Hilbert space to align all orders of moments, surpassing the optimization-oriented method IDC on high-resolution ImageNet.", "abstract": "Training state-of-the-art (SOTA) deep models often requires extensive data, resulting in substantial training and storage costs. To address these challenges, dataset condensation has been developed to learn a small synthetic set that preserves essential information from the original large-scale dataset. Nowadays, optimization-oriented methods have been the primary method in the field of dataset condensation for achieving SOTA results. However, the bi-level optimization process hinders the practical application of such methods to realistic and larger datasets. To enhance condensation efficiency, previous works proposed Distribution-Matching (DM) as an alternative, which significantly reduces the condensation cost. Nonetheless, current DM-based methods still yield less comparable results to SOTA optimization-oriented methods. In this paper, we argue that existing DM-based methods overlook the higher-order alignment of the distributions, which may lead to sub-optimal matching results. Inspired by this, we present a novel DM-based method named M3D for dataset condensation by Minimizing the Maximum Mean Discrepancy between feature representations of the synthetic and real images. By embedding their distributions in a reproducing kernel Hilbert space, we align all orders of moments of the distributions of real and synthetic images, resulting in a more generalized condensed set. Notably, our method even surpasses the SOTA optimization-oriented method IDC on the high-resolution ImageNet dataset. Extensive analysis is conducted to verify the effectiveness of the proposed method. Source codes are available at https://github.com/Hansong-Zhang/M3D.", "explained": { "before": "dm-2023-family methods (including idm-2023's class-aware regularization) align only the mean of the feature distribution; optimization-oriented methods such as idc-2022 pay a heavy bi-level cost, reportedly around 4 days for ImageNet-10 at IPC=20.", "problem": "Matching only the mean leaves higher-order statistics (variance, skewness, and beyond) unmatched, so real and synthetic feature distributions can still differ substantially even once their means agree.", "idea": "Embed features into a reproducing kernel Hilbert space and minimize the Maximum Mean Discrepancy there, which implicitly aligns all orders of moments rather than just the first, without training any network.", "method": "Optimizes synthetic pixel images against a class-wise MMD loss computed in an RKHS (Gaussian RBF kernel by default; polynomial/linear kernels give similar results) using several randomly initialized, untrained encoder networks. No bi-level loop and no classification loss -- the MMD term is the sole objective. For the high-resolution ImageNet experiments, M3D is combined with idc-2022's downsampled-multiformation parameterization (factor l=3) rather than plain pixels.", "evidence": "CIFAR-10, IPC=10: 63.5% vs. DM 58.6% (Table 2). CIFAR-10, IPC=50: 69.9%, on par with IDM's 69.9%. ImageNet-10, IPC=10 (ResNetAP-10): 73.4% vs. IDC 72.8%, DM 52.3% (Table 3); IPC=20: 76.8% vs. IDC 76.6%. ImageNet-100, IPC=10: 46.9% vs. IDC 46.7%; IPC=20: 55.5% vs. IDC 53.7% -- the first DM-family result to surpass the optimization-oriented IDC at ImageNet scale. Cost (Table 3): ImageNet-10 IPC=10, M3D 1.1h vs. DM 1.4h vs. IDC 70.1h; IPC=20, M3D 3.1h vs. IDC 92.8h (about 30x faster than IDC, slightly faster than DM). Ablation (Table 1, CIFAR-10, IPC=10): first/second/third-order moment distances shrink from DM's 4.91/7.37/6.76 to M3D's 0.82/1.23/1.64 as accuracy rises from 48.9% to 63.5%, directly linking higher-moment alignment to the accuracy gain.", "limitations": "No full ImageNet-1K evaluation is reported, only subsets of up to 100 classes. Kernel choice is ablated only lightly (roughly a 1-point spread across Gaussian/polynomial/linear). The ImageNet-scale results combine M3D's MMD objective with IDC's multi-formation parameterization, so the isolated contribution of higher-moment matching at that scale is entangled with the parameterization gain rather than cleanly separated." }, "builds_on": [ "dm-2023", "idm-2023", "idc-2022" ], "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ] }, { "id": "dcfl-2024", "type": "paper", "title": "DCFL: Non-IID awareness Data Condensation aided Federated Learning", "short": "DCFL", "arxiv": "2312.14219", "url": null, "date": "2023-12-21", "venue": { "name": "IJCNN 2024", "status": "conference" }, "authors": [ "Shaohan Sha", "YaFeng Sun" ], "kind": "application", "settings": [ "image-classification" ], "families": [], "applications": [ "federated-learning" ], "tier": "catalogued", "links": { "code": "https://github.com/JLUssh/DCFL", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Groups federated clients by Centered Kernel Alignment similarity, then uses non-IID-aware dataset condensation so that clients within the same group share each other's condensed data as complementary information; CKA-guided client selection, filtering and data-enhancement techniques are added to make use of the condensed data efficiently. Reports competitive performance on MNIST, FashionMNIST, SVHN and CIFAR-10 against existing FL protocols.", "abstract": "Federated learning is a decentralized learning paradigm wherein a central server trains a global model iteratively by utilizing clients who possess a certain amount of private datasets. The challenge lies in the fact that the client side private data may not be identically and independently distributed, significantly impacting the accuracy of the global model. Existing methods commonly address the Non-IID challenge by focusing on optimization, client selection and data complement. However, most approaches tend to overlook the perspective of the private data itself due to privacy constraints. Intuitively, statistical distinctions among private data on the client side can help mitigate the Non-IID degree. Besides, the recent advancements in dataset condensation technology have inspired us to investigate its potential applicability in addressing Non-IID issues while maintaining privacy. Motivated by this, we propose DCFL which divides clients into groups by using the Centered Kernel Alignment (CKA) method, then uses dataset condensation methods with non-IID awareness to complete clients. The private data from clients within the same group is complementary and their condensed data is accessible to all clients in the group. Additionally, CKA-guided client selection strategy, filtering mechanisms, and data enhancement techniques are incorporated to efficiently and precisely utilize the condensed data, enhance model performance, and minimize communication time. Experimental results demonstrate that DCFL achieves competitive performance on popular federated learning benchmarks including MNIST, FashionMNIST, SVHN, and CIFAR-10 with existing FL protocol.", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "Corrected authors and abstract from the arXiv version (2312.14219), which the draft had missed (draft listed arxiv: null with a placeholder author)." }, { "id": "dataset-distillation-via-adversarial-2023", "type": "paper", "title": "Dataset Distillation via Adversarial Prediction Matching", "arxiv": "2312.08912", "url": null, "date": "2023-12-14", "venue": { "name": "arXiv 2023", "status": "preprint" }, "authors": [ "Mingyang Chen", "Bo Huang", "Junda Lu", "Bing Li", "Yi Wang", "Minhao Cheng", "Wei Wang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "facets": { "regime": "single-level" }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Minimizes the prediction discrepancy on real data between a model trained on the full dataset and one trained on the distilled set, using a single-level adversarial framework in place of nested bi-level optimization or long trajectory unrolling. Reports distilled sets at 10% of the original size reaching 94% of full-data accuracy on average across datasets including ImageNet-1K, at 2.5x less memory and 5x less runtime than the prior state of the art, with ImageNet-1K distillation possible in 6.5GB of GPU memory.", "abstract": "Dataset distillation is the technique of synthesizing smaller condensed datasets from large original datasets while retaining necessary information to persist the effect. In this paper, we approach the dataset distillation problem from a novel perspective: we regard minimizing the prediction discrepancy on the real data distribution between models, which are respectively trained on the large original dataset and on the small distilled dataset, as a conduit for condensing information from the raw data into the distilled version. An adversarial framework is proposed to solve the problem efficiently. In contrast to existing distillation methods involving nested optimization or long-range gradient unrolling, our approach hinges on single-level optimization. This ensures the memory efficiency of our method and provides a flexible tradeoff between time and memory budgets, allowing us to distil ImageNet-1K using a minimum of only 6.5GB of GPU memory. Under the optimal tradeoff strategy, it requires only 2.5$\\times$ less memory and 5$\\times$ less runtime compared to the state-of-the-art. Empirically, our method can produce synthetic datasets just 10% the size of the original, yet achieve, on average, 94% of the test accuracy of models trained on the full original datasets including ImageNet-1K, significantly surpassing state-of-the-art. Additionally, extensive tests reveal that our distilled datasets excel in cross-architecture generalization capabilities.", "notes": "Matches output predictions via a single-level adversarial objective rather than feature moments; tagged distribution-matching as the closest existing family — worth a second look." }, { "id": "boosting-the-cross-architecture-2023", "type": "paper", "title": "Boosting the Cross-Architecture Generalization of Dataset Distillation through an Empirical Study", "short": "ELF", "arxiv": "2312.05598", "url": null, "date": "2023-12-09", "venue": { "name": "arXiv 2023", "status": "preprint" }, "authors": [ "Lirui Zhao", "Yuxin Zhang", "Fei Chao", "Rongrong Ji" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [], "tier": "catalogued", "links": { "code": "https://github.com/Lirui-Zhao/ELF", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Attributes the poor cross-architecture transfer of distilled datasets to an inductive bias the synthetic images inherit from the network used to distill them, then proposes ELF, an evaluation procedure that trains the evaluation network against intermediate features of the distillation model instead of only labels. Reports that ELF improves cross-architecture accuracy for several existing distillation methods without changing the distilled images themselves.", "abstract": "The poor cross-architecture generalization of dataset distillation greatly weakens its practical significance. This paper attempts to mitigate this issue through an empirical study, which suggests that the synthetic datasets undergo an inductive bias towards the distillation model. Therefore, the evaluation model is strictly confined to having similar architectures of the distillation model. We propose a novel method of EvaLuation with distillation Feature (ELF), which utilizes features from intermediate layers of the distillation model for the cross-architecture evaluation. In this manner, the evaluation model learns from bias-free knowledge therefore its architecture becomes unfettered while retaining performance. By performing extensive experiments, we successfully prove that ELF can well enhance the cross-architecture generalization of current DD methods. Code of this project is at \\url{https://github.com/Lirui-Zhao/ELF}.", "notes": "Proposes an evaluation-time fix rather than a synthesis mechanism; no family applies." }, { "id": "rded-2024", "type": "paper", "title": "On the Diversity and Realism of Distilled Dataset: An Efficient Dataset Distillation Paradigm", "short": "RDED", "arxiv": "2312.03526", "url": null, "date": "2023-12-06", "venue": { "name": "CVPR 2024", "status": "conference" }, "authors": [ "Peng Sun", "Bei Shi", "Daiwei Yu", "Tao Lin" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "tier": "core", "links": { "code": "https://github.com/LINs-lab/RDED", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "labels": "soft-relabel", "prior": [ "pretrained-classifier" ], "regime": "selection", "parameterization": [ "pixels" ], "scale": [ "imagenet-1k", "tiny-imagenet", "mnist-cifar" ] }, "explained": { "before": "Groups prior work into bi-level optimization (DC/DSA/MTT), prior-regularized bi-level optimization (GLaD), uni-level/decoupled optimization (SRe2L) and coreset selection (Herding), and evaluates each on realism, diversity and efficiency using qualitative image comparisons (Fig. 2) plus each category's own reported costs.", "problem": "Bi-level methods produce noise-like, unrealistic images and overfit to the synthesis architecture; prior-regularized bi-level methods (GLaD) fix realism but inherit bi-level's poor efficiency; SRe2L is efficient and more realistic but, having distilled a pretrained model containing only partial information about the dataset, has limited diversity; coreset selection is realistic but too low in information/diversity. No single method satisfies realism, diversity and efficiency together.", "idea": "Treat distillation as maximizing V-information rather than optimizing pixels: score real-image crops for class-informativeness with a pretrained observer model, keep a diverse top-scoring subset, and spatially stitch several crops into one synthetic image with region-level soft labels — no pixel gradients are computed at all.", "method": "An observer model $\\varphi_{\\theta_{\\mathcal T}}$ (ResNet-18 for ImageNet-scale sets, a modified ResNet-18 or ConvNet for CIFAR/Tiny-ImageNet) scores random crops of real images by negative cross-entropy against the true label. Per class, 300 images are pre-selected, each split into $K{=}5$ random patches, the single best-scoring patch per image kept (300 candidate patches/class), then the top $N\\times\\text{IPC}$ patches by score are retained. $N$ patches (4 for high-resolution images in a 2x2 grid, 1 for CIFAR-scale) are drawn without replacement and concatenated into one synthetic image. No BN-statistics matching and no optimization loop is used; the method cannot even be applied to architectures without batch-norm the way SRe2L can. Relabeling stores an FKD-style soft label per region/patch within a stitched image rather than one label per image. Post-eval augmentation follows SRe2L (RandomResizedCrop + CutMix); students evaluated include ResNet-18/101, EfficientNet-B0, MobileNetV2, VGG-11, Swin-V2-Tiny, and ConvNet-3/4 for CIFAR/Tiny-ImageNet.", "evidence": "ImageNet-1K, ResNet-18 (Table 2): IPC=10, RDED 42.0% vs SRe2L 21.3%; IPC=50, 56.5% vs 46.8%. ResNet-101: IPC=10 48.3% vs 30.9%; IPC=50 61.2% vs 60.8%. Tiny-ImageNet, ResNet-18: IPC=10 41.9% vs SRe2L 16.1%; IPC=50 58.2% vs 41.1%. On small ConvNet-distilled CIFAR-10, RDED is weaker than trajectory matching (IPC=10 50.2% vs MTT 65.3%; IPC=50 68.4% vs DATM 76.1%), but stronger on CIFAR-100 (IPC=10 48.1% vs MTT 40.1%). Synthesis cost (Table 3, one RTX-4090, 100 images at once, ResNet-18): 39.89ms/image and 1.57GB peak memory for RDED vs 2113.23ms/image and 9.14GB for SRe2L — the \"10 IPC of ImageNet-1K in 7 minutes vs 6 hours, 42% vs 21%\" headline claim. No hard-label result is reported; all comparisons use soft/relabeled targets. Key ablation (Table 9): realism-score selection beats Random/Herding/K-Means selection on every one of 6 datasets (e.g. ImageNet-1K 42.0% vs Random 37.9%, Herding 38.4%, K-Means 38.2%), isolating the scoring mechanism itself as the source of gain, independent of the stitching step.", "limitations": "States the method struggles at IPC=1 on both high- and low-resolution datasets and its efficacy diminishes on smaller datasets generally. Depends on a pretrained observer model trained on the full real dataset and on that dataset containing enough real, well-scoring patches — a reliance on real data availability and diversity that a later critique (Hard truths about soft labels) ties directly to both RDED's strength under hard labels and its limits. No formal limitations section; these points are embedded in the experiments discussion rather than argued explicitly." }, "builds_on": [], "summary": "Identifies realism, diversity and efficiency as the properties large-scale dataset distillation needs and shows existing methods trade off against them; RDED selects and stitches realistic real-image crops scored by a teacher rather than optimizing synthetic pixels, distilling all of ImageNet-1K to 10 images per class in 7 minutes and reaching 42% top-1 with ResNet-18 on a single RTX-4090, versus 21% in 6 hours for the prior state of the art.", "abstract": "Contemporary machine learning requires training large neural networks on massive datasets and thus faces the challenges of high computational demands. Dataset distillation, as a recent emerging strategy, aims to compress real-world datasets for efficient training. However, this line of research currently struggle with large-scale and high-resolution datasets, hindering its practicality and feasibility. To this end, we re-examine the existing dataset distillation methods and identify three properties required for large-scale real-world applications, namely, realism, diversity, and efficiency. As a remedy, we propose RDED, a novel computationally-efficient yet effective data distillation paradigm, to enable both diversity and realism of the distilled data. Extensive empirical results over various neural architectures and datasets demonstrate the advancement of RDED: we can distill the full ImageNet-1K to a small dataset comprising 10 images per class within 7 minutes, achieving a notable 42% top-1 accuracy with ResNet-18 on a single RTX-4090 GPU (while the SOTA only achieves 21% but requires 6 hours).", "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "unlocking-the-potential-of-2024", "type": "paper", "title": "Unlocking the Potential of Federated Learning: The Symphony of Dataset Distillation via Deep Generative Latents", "short": "FedDG", "arxiv": "2312.01537", "url": null, "date": "2023-12-03", "venue": { "name": "ECCV 2024", "status": "conference" }, "authors": [ "Yuqi Jia", "Saeed Vahidian", "Jingwei Sun", "Jianyi Zhang", "Vyacheslav Kungurtsev", "Neil Zhenqiang Gong", "Yiran Chen" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "generative-latent-optimization" ], "applications": [ "federated-learning" ], "tier": "notable", "links": { "code": "https://github.com/FedDG23/FedDG-main", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Moves dataset distillation for federated learning from the client to the server: instead of clients distilling and uploading synthetic data, the server uses prior knowledge from pretrained deep generative models' latents to synthesize data representations from heterogeneous client model architectures, letting local devices train smaller surrogate models while the server trains a larger global model on a multi-modal distribution. Reports up to 40% accuracy improvement over non-distillation FL and 18% over existing dataset-distillation FL methods under high heterogeneity, plus faster convergence, with a theoretical argument that the process asymptotically resembles centralized training.", "abstract": "Data heterogeneity presents significant challenges for federated learning (FL). Recently, dataset distillation techniques have been introduced, and performed at the client level, to attempt to mitigate some of these challenges. In this paper, we propose a highly efficient FL dataset distillation framework on the server side, significantly reducing both the computational and communication demands on local devices while enhancing the clients' privacy. Unlike previous strategies that perform dataset distillation on local devices and upload synthetic data to the server, our technique enables the server to leverage prior knowledge from pre-trained deep generative models to synthesize essential data representations from a heterogeneous model architecture. This process allows local devices to train smaller surrogate models while enabling the training of a larger global model on the server, effectively minimizing resource utilization. We substantiate our claim with a theoretical analysis, demonstrating the asymptotic resemblance of the process to the hypothetical ideal of completely centralized training on a heterogeneous dataset. Empirical evidence from our comprehensive experiments indicates our method's superiority, delivering an accuracy enhancement of up to 40% over non-dataset-distillation techniques in highly heterogeneous FL contexts, and surpassing existing dataset-distillation methods by 18%. In addition to the high accuracy, our framework converges faster than the baselines because rather than the server trains on several sets of heterogeneous data distributions, it trains on a multi-modal distribution. Our code is available at https://github.com/FedDG23/FedDG-main.git", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "" }, { "id": "video-sdd-2024", "type": "paper", "title": "Dancing with Still Images: Video Distillation via Static-Dynamic Disentanglement", "short": "Static-dynamic video DD", "arxiv": "2312.00362", "url": null, "date": "2023-12-01", "venue": { "name": "CVPR 2024", "status": "conference" }, "authors": [ "Ziyu Wang", "Yue Xu", "Cewu Lu", "Yong-Lu Li" ], "kind": "method", "settings": [ "video" ], "families": [ "gradient-matching", "parameterization" ], "facets": { "parameterization": [ "addressable-memory" ] }, "tier": "core", "links": { "code": "https://github.com/yuz1wan/video_distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "This paper gives the first systematic study of video distillation, finding that the temporal dimension of synthetic clips is usually not well learned and contributes little, and responds by disentangling static and dynamic information: a static learning stage distills videos into still images via gradient matching, and a dynamic fine-tuning stage compensates motion with a separately stored, learnable dynamic memory block rather than distilling every frame. The paper reports state-of-the-art results on video datasets at multiple scales with a notably smaller memory storage budget than prior methods.", "abstract": "Recently, dataset distillation has paved the way towards efficient machine learning, especially for image datasets. However, the distillation for videos, characterized by an exclusive temporal dimension, remains an underexplored domain. In this work, we provide the first systematic study of video distillation and introduce a taxonomy to categorize temporal compression. Our investigation reveals that the temporal information is usually not well learned during distillation, and the temporal dimension of synthetic data contributes little. The observations motivate our unified framework of disentangling the dynamic and static information in the videos. It first distills the videos into still images as static memory and then compensates the dynamic and motion information with a learnable dynamic memory block. Our method achieves state-of-the-art on video datasets at different scales, with a notably smaller memory storage budget. Our code is available at https://github.com/yuz1wan/video_distillation.", "notes": "Per secondary sources, the dynamic fine-tuning stage can also use trajectory or distribution matching; gradient-matching is listed as the primary/static-stage backbone.", "explained": { "before": "Naive video distillation directly ports image-domain matching (DC's gradient matching, DM's distribution matching, MTT's trajectory matching, FRePo's kernel closed-form regression) to video by treating every synthetic clip as a small stack of frames matched against a bipartite set of real frames, with no special handling of the time axis.", "problem": "The paper's own pre-analysis (Sec. 3) shows this naive approach wastes most of its budget: increasing the number of real/synthetic frames matched, or the number of temporal segments, gives at most a few points of accuracy while multiplying compute and memory (16x GPU memory going from 1 to 16 frames), and distilling to a single still frame per class already recovers most of the achievable accuracy — meaning the temporal dimension of naively distilled synthetic clips contributes little of what is learned.", "idea": "Disentangle what a synthetic clip has to represent into a static component (appearance, well suited to image-style distillation) and a dynamic component (motion, cheap to represent compactly), distill each separately, and recombine them with a small learned integrator network rather than distilling every frame jointly.", "method": "Stage 1 (static learning): treat one randomly sampled frame per video, per epoch, as an image dataset and run DC-style gradient matching on a ConvNet to learn a static memory $\\mathcal{S}$ (one or more frames per class); this reduces the video problem to $N_{syn}{=}N_{real}{=}K{=}1$ in the paper's \"segmented matching and interpolation\" formalism. Stage 2 (dynamic fine-tuning): freeze $\\mathcal{S}$, initialize a dynamic memory $\\mathcal{D}$ (single-channel frames) and an integrator network $\\mathcal{H}$ from noise, and jointly optimize both by applying an existing distillation matching loss $\\mathcal{A}$ (DM, MTT, or FRePo, used as plugins) between $\\mathcal{H}(\\mathcal{D},\\mathcal{S})$ (reconstructed synthetic clips) and multi-frame samples of the real videos. Stored artifact = static memory + dynamic memory + $\\mathcal{H}$'s weights, kept under the same total-tensor storage budget as the naive baseline (82% of baseline storage for DM/MTT, 42% for FRePo).", "evidence": "Table 2: MiniUCF (50-class UCF101 subset) IPC=1, ConvNet3D eval, hard labels — FRePo+Ours 22.0% vs. FRePo alone 20.3% (full data 57.2%) using ≤42% of FRePo's storage; MTT+Ours 23.3% vs. MTT alone 19.0% at 82% storage. HMDB51 IPC=1: FRePo+Ours 8.6% vs. FRePo 7.2% (full data 28.6%). Table 3 (top-5, large-scale): Kinetics-400 IPC=5, MTT+Ours 11.5% vs. MTT 9.1% (full 34.6%); SSv2 IPC=1, MTT+Ours 5.5% vs. MTT 3.9% (full 29.0%). Table 4 (cross-architecture, MiniUCF IPC=1): MTT+Ours transfers much better to unseen CNN+GRU (14.8% vs. 8.4%) and CNN+LSTM (13.4% vs. 7.3%) architectures than naive MTT. Key ablation (Table 1, Sec. 3.3): distilling to a single still frame per class already reaches >17% accuracy on MiniUCF with DM, and increasing real/synthetic frame counts from 1 to 8 gains under 3 points while using up to 16x more GPU memory, directly motivating the disentangled design; a second ablation (Fig. 8, Table 8) shows the disentangled method specifically improves accuracy on the \"dynamic\" (high inter-frame-change) half of MiniUCF classes relative to a \"Static-DC\" (frame duplication only) baseline, isolating that the dynamic memory is capturing real motion information.", "limitations": "Gains shrink on the largest, most class-rich dataset (Kinetics-400, 400 classes) because the integrator network $\\mathcal{H}$ is shared across all classes and the paper explicitly declines to scale up per-class capacity due to training cost; only short, low-resolution clips are tested (16 frames at 112x112 for small datasets, 8 frames at 64x64 for Kinetics/SSv2); evaluation is restricted to action recognition with fairly small backbones (MiniC3D, CNN+GRU/LSTM), not modern video transformers; FRePo's own results are flagged by the authors as \"reference only\" because FRePo does its own label learning and uses a different optimizer, complicating direct comparison." } }, { "id": "a-theoretical-study-of-2023", "type": "paper", "title": "A Theoretical Study of Dataset Distillation", "arxiv": null, "url": "https://openreview.net/forum?id=dq5QGXGxoJ", "date": "2023-12", "venue": { "name": "NeurIPS 2023 Workshop", "status": "workshop" }, "authors": [ "Zachary Izzo", "James Zou" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "openreview", "depth": "abstract" }, "summary": "Proves existence and impossibility results for exact dataset distillation of generalized linear models: a single synthetic point can exactly reproduce a model trained on the full data for GLMs, and a size-independent construction exists for linear regression with any data-independent regularizer, but no analogous construction exists for logistic regression, and kernel regression cannot in general be distilled to recover even one model exactly.", "abstract": null, "awesome_sections": [ "Main / Better Understanding" ], "notes": "Non-arXiv; abstract read via OpenReview/ML Anthology, not stored verbatim per policy." }, { "id": "wmdd-2025", "type": "paper", "title": "Dataset Distillation via the Wasserstein Metric", "short": "WMDD", "arxiv": "2311.18531", "url": null, "date": "2023-11-30", "venue": { "name": "ICCV 2025", "status": "conference" }, "authors": [ "Haoyang Liu", "Yijiang Li", "Tiancheng Xing", "Peiran Wang", "Vibhu Dalal", "Luwei Li", "Jingrui He", "Haohan Wang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "tier": "notable", "links": { "code": "https://github.com/Liu-Hy/WMDD", "project": "https://liu-hy.github.io/WMDD/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "parameterization": [ "pixels" ], "labels": "hard", "prior": [ "pretrained-classifier" ], "regime": "single-level", "scale": [ "imagenet-subsets", "imagenet-1k" ] }, "explained": { "before": "dm-2023-style mean matching and linear-kernel MMD are argued to be \"not characteristic\" and to neglect higher-order moments; at ImageNet scale, distribution-matching accuracy has lagged far behind decoupled BN-statistics-inversion methods such as sre2l-2023 and g-vbsm-2024.", "problem": "Mean/MMD-based matching produces what the paper calls a \"rigid mix-up\" of the real feature distribution rather than a natural interpolation of it, and existing distribution-matching methods trail decoupled methods badly at ImageNet-1K scale.", "idea": "Compute the Wasserstein barycenter of real features from a pretrained classifier -- which optimally interpolates a distribution rather than crudely averaging it -- and optimize synthetic images to match that barycenter in feature space, restoring the intra-class variation a single barycenter would erase by matching BatchNorm statistics separately per class.", "method": "Extracts penultimate-layer features of a classifier (ResNet-18/-50) pretrained once on the full real dataset (a \"squeeze stage\" akin to sre2l-2023's teacher); computes the Wasserstein barycenter of each class's real features via alternating optimal transport (about 10 iterations: subgradient descent on transport weights, a Newton step on barycenter positions); optimizes synthetic pixel images with a single-level loss combining an L2 distance to the barycenter features and a per-class BatchNorm-statistics term (BN running mean/variance computed separately per class rather than globally, so images keep the class's spread instead of collapsing to the barycenter's single point). No diffusion or other generative model is used, and there is no bi-level loop.", "evidence": "ImageNet-1K, ResNet-18 eval: IPC=10, 38.2% vs. SRe2L 21.3%, G-VBSM 31.4% (Table 1); IPC=50, 57.6% vs. SRe2L 46.8%, G-VBSM 51.8%; IPC=100, 60.7% vs. the full-dataset ceiling of 63.1%. ImageNette, IPC=10: 64.8% vs. SRe2L 54.2%, DataDAM 59.4%. Cross-architecture (ImageNet-1K, IPC=50): ResNet-18 57.83%, ResNet-50 61.22%, ViT-Tiny 34.25% (markedly weaker on ViT). Cost (Table 3, ImageNette, IPC=1, RTX 3090): WMDD 0.013s/iter, 1.22GB, 207.5s total vs. SRe2L 0.015s/1.14GB/194.9s (comparable, since both use a single pretrained teacher rather than sampling many random networks per step) vs. DM 1.965s/9.93GB/4018s and DC 2.154s/11.90GB/6348s (18-30x more expensive). Ablation (Table 4, IPC=10): Wasserstein loss + per-class BN is best on every dataset (ImageNette 64.7%, TinyImageNet 41.8%, ImageNet-1K 38.1%); dropping per-class BN costs 4-11pp; replacing the Wasserstein term with vanilla MMD gives \"near-random performance\" (Figure 3).", "limitations": "Requires a classifier pretrained on the full real dataset, tying the method's accuracy ceiling to that classifier's feature quality -- a real, if amortizable, cost not present in random-network distribution matching. Performance on Vision Transformers is notably weaker (\"probably due to their data-hungry property,\" per the paper). Not tested outside vision. The paper acknowledges the method \"inherently reflects existing biases\" in the pretrained classifier's training data. The label regime used at evaluation is not fully specified in the sections read; given the SRe2L-style pipeline it is compared against, soft/relabeled labels are plausible but not confirmed here." }, "summary": "Computes the Wasserstein barycenter of pretrained-classifier features from the real data and optimizes synthetic images to align with this barycenter in feature space, combined with per-class batch-norm statistics to preserve intra-class variation, reporting state-of-the-art results across several high-resolution dataset-distillation benchmarks while keeping the efficiency of distribution-matching approaches.", "abstract": "Dataset Distillation (DD) aims to generate a compact synthetic dataset that enables models to achieve performance comparable to training on the full large dataset, significantly reducing computational costs. Drawing from optimal transport theory, we introduce WMDD (Wasserstein Metric-based Dataset Distillation), a straightforward yet powerful method that employs the Wasserstein metric to enhance distribution matching. We compute the Wasserstein barycenter of features from a pretrained classifier to capture essential characteristics of the original data distribution. By optimizing synthetic data to align with this barycenter in feature space and leveraging per-class BatchNorm statistics to preserve intra-class variations, WMDD maintains the efficiency of distribution matching approaches while achieving state-of-the-art results across various high-resolution datasets. Our extensive experiments demonstrate WMDD's effectiveness and adaptability, highlighting its potential for advancing machine learning applications at scale.", "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ], "builds_on": [ "dm-2023", "sre2l-2023", "g-vbsm-2024" ] }, { "id": "cda-2024", "type": "paper", "title": "Dataset Distillation via Curriculum Data Synthesis in Large Data Era", "short": "CDA", "arxiv": "2311.18838", "url": null, "date": "2023-11-30", "venue": { "name": "TMLR 2024", "status": "journal" }, "authors": [ "Zeyuan Yin", "Zhiqiang Shen" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "tier": "notable", "links": { "code": "https://github.com/VILA-Lab/SRe2L/tree/main/CDA", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "labels": "soft-relabel", "prior": [ "pretrained-classifier" ], "regime": "decoupled", "parameterization": [ "pixels" ], "scale": [ "imagenet-1k", "imagenet-21k", "tiny-imagenet" ] }, "explained": { "before": "Names SRe2L as its \"closest baseline\" and keeps its squeeze-recover-relabel pipeline and its cross-entropy-plus-BN-statistics recovery loss unchanged. Also compares against MTT and TESLA (trajectory matching) and DM/DSA/FRePo (surrogate/kernel methods) on small-scale data.", "problem": "SRe2L uses a constant RandomResizedCrop range across all recovery iterations even though, the paper argues and visualizes, the first few update steps already fix an image's global outline; a poor early crop distribution permanently bakes a noisy or degenerate layout into the synthetic image, and the later iterations can only refine local detail around that fixed outline.", "idea": "Schedule the crop scale used during recovery from global to local: start with large, near-full-image crops so the outline forms correctly, then shrink the minimum crop scale over iterations (a curriculum, step/linear/cosine schedules tested, cosine chosen) so later iterations refine local texture. No new loss term is added — only the augmentation schedule fed into the unchanged SRe2L objective changes.", "method": "Squeeze uses the same off-the-shelf teachers as SRe2L (ResNet-18/DenseNet-121 for ImageNet-1K); for ImageNet-21K a ResNet-18/50 is initialized from ImageNet-1K weights and fine-tuned with label smoothing and stronger augmentation. Recover keeps SRe2L's loss (cross-entropy against the target label plus BN mean/variance matching) but replaces the fixed RandomResizedCrop range with a schedule $\\alpha=\\beta_l+\\gamma\\cdot(\\beta_u+\\cos(\\pi s/T))/2$ moving the minimum crop scale from near 1.0 toward 0.08 as iteration $s$ approaches milestone $T$. Relabel is unchanged from SRe2L: FKD-style stored soft labels from the same teacher, no per-epoch recomputation. Post-evaluation uses AdamW, cosine LR, 300 epochs, RandomResizedCrop only (no CutMix/Mixup), and a smaller batch size (32) than SRe2L's, justified via a generalization-gap argument.", "evidence": "ImageNet-1K, ResNet-18, IPC=50 (Table 1/4): CDA 53.5% vs SRe2L 46.8% (both soft-labeled); IPC=100 58.0% vs 52.8%; IPC=200 63.3% vs 57.0%. The headline \"63.2% at IPC=50\" and \"36.1% at IPC=20\" numbers in the abstract are not the ResNet-18 figures but the best cross-architecture results (RegNet-Y-8GF on ImageNet-1K; ResNet-101 on ImageNet-21K). First ImageNet-21K distillation at 224x224 (Table 4): IPC=20, ResNet-18 26.4% vs SRe2L 21.8% (both replicated by CDA, since SRe2L never reported 21K); ResNet-101 36.1% vs 33.2%. Table 1 marks several SRe2L numbers with an asterisk as CDA's own replication (e.g. ImageNet-1K IPC=10 SRe2L* 21.3%) where SRe2L's own paper did not report that setting, while IPC=50/100/200 SRe2L numbers are taken verbatim from SRe2L's paper. No hard-label result is reported. Convergence claim (Table 17): CDA at 1K recovery iterations (52.9%, ResNet-18) beats SRe2L's full 4K-iteration budget (46.8%). Key ablation (Table 3): fixed, non-curriculum crop ranges degrade sharply at either extreme (large-only or small-only crops) relative to the reproduced SRe2L baseline (44.9%), isolating the global-to-local schedule itself as the mechanism.", "limitations": "States synthetic realism is not competitive with generative models by design, since the objective prioritizes informativeness. Reports no advantage at very low IPC (e.g. IPC=1). Not evaluated beyond ImageNet-21K. Still fully dependent on SRe2L's teacher-architecture-specific BN-matching recovery objective — CDA changes only the augmentation schedule, not the underlying loss — and its curriculum hyperparameters ($\\beta_l,\\beta_u,\\gamma$, milestone) are tuned per dataset and architecture, an unaddressed tuning burden." }, "builds_on": [ "sre2l-2023" ], "summary": "Extends SRe2L's decoupled synthesis with curriculum data augmentation that refines the gradient update from global to local over the synthesis iterations, since the first few update steps determine the final image outline; reports 63.2% top-1 on ImageNet-1K at IPC 50 and 36.1% on ImageNet-21K at IPC 20, the first reported ImageNet-21K distillation at 224x224 resolution, beating SRe2L, TESLA and MTT by more than 4 points.", "abstract": "Dataset distillation or condensation aims to generate a smaller but representative subset from a large dataset, which allows a model to be trained more efficiently, meanwhile evaluating on the original testing data distribution to achieve decent performance. Previous decoupled methods like SRe$^2$L simply use a unified gradient update scheme for synthesizing data from Gaussian noise, while, we notice that the initial several update iterations will determine the final outline of synthesis, thus an improper gradient update strategy may dramatically affect the final generation quality. To address this, we introduce a simple yet effective global-to-local gradient refinement approach enabled by curriculum data augmentation ($\\texttt{CDA}$) during data synthesis. The proposed framework achieves the current published highest accuracy on both large-scale ImageNet-1K and 21K with 63.2% under IPC (Images Per Class) 50 and 36.1% under IPC 20, using a regular input resolution of 224$\\times$224 with faster convergence speed and less synthetic time. The proposed model outperforms the current state-of-the-art methods like SRe$^2$L, TESLA, and MTT by more than 4% Top-1 accuracy on ImageNet-1K/21K and for the first time, reduces the gap to its full-data training counterparts to less than absolute 15%. Moreover, this work represents the inaugural success in dataset distillation on the larger-scale ImageNet-21K dataset under the standard 224$\\times$224 resolution. Our code and distilled ImageNet-21K dataset of 20 IPC, 2K recovery budget are available at https://github.com/VILA-Lab/SRe2L/tree/main/CDA.", "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "g-vbsm-2024", "type": "paper", "title": "Generalized Large-Scale Data Condensation via Various Backbone and Statistical Matching", "short": "G-VBSM", "arxiv": "2311.17950", "url": null, "date": "2023-11-29", "venue": { "name": "CVPR 2024", "status": "conference" }, "authors": [ "Shitong Shao", "Zeyuan Yin", "Muxin Zhou", "Xindong Zhang", "Zhiqiang Shen" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "tier": "notable", "links": { "code": "https://github.com/shaoshitong/G_VBSM_Dataset_Condensation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "labels": "soft-relabel", "prior": [ "pretrained-classifier" ], "regime": "decoupled", "parameterization": [ "pixels" ], "scale": [ "imagenet-1k", "tiny-imagenet", "mnist-cifar" ] }, "explained": { "before": "Builds directly on SRe2L, whose recovery matches BN mean/variance of a single backbone (ResNet-18) against a single synthetic-batch statistic — a \"local-match-global\" scheme the paper calls \"one-sided.\" Also compares against MTT/FTD (trajectory matching), DC (gradient matching) and DM/CAFE (distribution matching), which it says never scale past small or downsampled ImageNet.", "problem": "A single backbone, single layer type (BN only) and single statistic (channel mean/variance) cannot capture enough of the real distribution to transfer well to unseen evaluation architectures, and SRe2L underperforms classical distribution/gradient-matching methods on small datasets like CIFAR-100 despite winning on ImageNet-1K.", "idea": "\"Generalized matching\": perform many local-match-global matches at once — across several pretrained backbones, both BN and convolutional layers, and both channel- and patch-level statistics — plus an explicit diversity loss over each class's feature Gram matrix, so no single backbone or statistic determines the synthetic data.", "method": "Squeeze uses 4 ImageNet-1K backbones (ResNet-18, MobileNetV2, EfficientNet-B0, ShuffleNetV2-0.5, torchvision weights) or 5 backbones trained from scratch for CIFAR/Tiny-ImageNet. Recover matches, via an EMA/score-distillation-style update, both BN statistics ($L'_{BN}$) and convolutional-layer channel- and patch-level mean/variance ($L'_{Conv}$, patch count 4-16 depending on dataset), adds a Data Densification loss (KL divergence over softmaxed eigenvalues of each class's batch Gram matrix, forcing full-rank/diverse batches), and randomly samples one backbone per iteration from the candidate set (Generalized Backbone Matching) rather than running all backbones in parallel. Relabel averages logits across all candidate backbones (optionally Frobenius-normalized) into a single ensemble soft label per crop, stored via FKD. Evaluation replaces SRe2L's KL loss with MSE plus a small one-hot cross-entropy term (weight 0.1), argued to be the $\\tau\\to\\infty$ limit of KL. Post-eval: ImageNet-1K/Tiny-ImageNet largely follow SRe2L's 300-epoch recipe; CIFAR-10/100 use 1000 epochs on a 128-width ConvNet to match prior DC baselines.", "evidence": "ImageNet-1K, ResNet-18 (Table 4): IPC=10, G-VBSM 31.4% vs SRe2L's own reported 21.3%; IPC=50, 51.8% vs 46.8%; ResNet-50/101 also improve at both budgets (e.g. IPC=50 ResNet-101: 61.0% vs 60.8%). Tiny-ImageNet IPC=50, ResNet-18: 47.6% vs SRe2L 41.1% (Table 4). CIFAR-100 IPC=10, 128-width ConvNet: 38.7% vs a G-VBSM-run SRe2L baseline of 31.6% (Table 5) — SRe2L never reported CIFAR results itself, so this is a first application rather than a re-run discrepancy; ImageNet-1K/Tiny-ImageNet SRe2L numbers are cited verbatim from SRe2L's own paper. Cross-architecture generalization at IPC=50 on unseen students (Table 2): DeiT-Tiny 29.4% vs SRe2L 15.4%; Swin-Tiny 57.4% vs 39.2%. Key ablation (Table 3), number of teacher backbones at IPC=10: 1 backbone 25.7%/30.2% (ResNet-18/50 eval) rising to 31.4%/34.5% with all 4 backbones — a +5.7/+4.3 point gain from ensembling backbones alone. No hard-label result is reported; all evaluations use soft or MSE+GT-distilled labels.", "limitations": "No dedicated limitations section. Parallel multi-backbone matching is acknowledged as \"extremely expensive,\" mitigated by random single-backbone sampling per iteration rather than solved; the extra convolutional-statistics matching adds computational burden, mitigated with a random dropout probability rather than removed; logit normalization helps small models but is shown to hurt ResNet-50/101 (Table 6), so it must be selectively toggled per architecture — the design space is not architecture-agnostic despite the paper's stated goal of generalization." }, "builds_on": [ "sre2l-2023" ], "summary": "Argues that SRe2L's single-backbone, single-layer 'local-match-global' matching limits generalization, and generalizes it to match densities across many backbones, layers and statistics at once; the first algorithm to perform well at both small and large scale, reporting 38.7% on CIFAR-100 (IPC 10), 47.6% on Tiny-ImageNet (IPC 50), and 31.4% on full 224x224 ImageNet-1K (IPC 10), beating prior state of the art by 3.9-10.1 points.", "abstract": "The lightweight \"local-match-global\" matching introduced by SRe2L successfully creates a distilled dataset with comprehensive information on the full 224x224 ImageNet-1k. However, this one-sided approach is limited to a particular backbone, layer, and statistics, which limits the improvement of the generalization of a distilled dataset. We suggest that sufficient and various \"local-match-global\" matching are more precise and effective than a single one and has the ability to create a distilled dataset with richer information and better generalization. We call this perspective \"generalized matching\" and propose Generalized Various Backbone and Statistical Matching (G-VBSM) in this work, which aims to create a synthetic dataset with densities, ensuring consistency with the complete dataset across various backbones, layers, and statistics. As experimentally demonstrated, G-VBSM is the first algorithm to obtain strong performance across both small-scale and large-scale datasets. Specifically, G-VBSM achieves a performance of 38.7% on CIFAR-100 with 128-width ConvNet, 47.6% on Tiny-ImageNet with ResNet18, and 31.4% on the full 224x224 ImageNet-1k with ResNet18, under images per class (IPC) 10, 50, and 10, respectively. These results surpass all SOTA methods by margins of 3.9%, 6.5%, and 10.1%, respectively.", "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "discovering-galaxy-features-via-2023", "type": "paper", "title": "Discovering Galaxy Features via Dataset Distillation", "arxiv": "2311.17967", "url": null, "date": "2023-11-29", "venue": { "name": "NeurIPS 2023 Workshop", "status": "workshop" }, "authors": [ "Haowen Guan", "Xuan Zhao", "Zishi Wang", "Zhiyang Li", "Julia Kempe" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "trajectory-matching" ], "applications": [ "scientific-discovery" ], "tier": "catalogued", "links": { "code": "https://github.com/HaowenGuan/Galaxy-Dataset-Distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Uses dataset distillation on a class-balanced Galaxy Zoo 2 subset to visualize what a galaxy-morphology classifier relies on, treating the synthesized prototypical images as human-inspectable summaries of the features a neural net uses to classify galaxy morphology. Introduces a self-adaptive variant of trajectory matching to automate the distillation process, reporting enhanced performance on standard computer-vision benchmarks as a byproduct.", "abstract": "In many applications, Neural Nets (NNs) have classification performance on par or even exceeding human capacity. Moreover, it is likely that NNs leverage underlying features that might differ from those humans perceive to classify. Can we \"reverse-engineer\" pertinent features to enhance our scientific understanding? Here, we apply this idea to the notoriously difficult task of galaxy classification: NNs have reached high performance for this task, but what does a neural net (NN) \"see\" when it classifies galaxies? Are there morphological features that the human eye might overlook that could help with the task and provide new insights? Can we visualize tracers of early evolution, or additionally incorporated spectral data? We present a novel way to summarize and visualize galaxy morphology through the lens of neural networks, leveraging Dataset Distillation, a recent deep-learning methodology with the primary objective to distill knowledge from a large dataset and condense it into a compact synthetic dataset, such that a model trained on this synthetic dataset achieves performance comparable to a model trained on the full dataset. We curate a class-balanced, medium-size high-confidence version of the Galaxy Zoo 2 dataset, and proceed with dataset distillation from our accurate NN-classifier to create synthesized prototypical images of galaxy morphological features, demonstrating its effectiveness. Of independent interest, we introduce a self-adaptive version of the state-of-the-art Matching Trajectory algorithm to automate the distillation process, and show enhanced performance on computer vision benchmarks.", "awesome_sections": [ "Applications / Fashion, Art, and Design" ], "notes": "awesome_sections retained from the source list, though the application is scientific discovery (astronomy) rather than fashion/art/design." }, { "id": "rethinking-backdoor-attacks-on-2024", "type": "paper", "title": "Rethinking Backdoor Attacks on Dataset Distillation: A Kernel Method Perspective", "arxiv": "2311.16646", "url": null, "date": "2023-11-28", "venue": { "name": "ICLR 2024", "status": "conference" }, "authors": [ "Ming-Yu Chung", "Sheng-Yen Chou", "Chia-Mu Yu", "Pin-Yu Chen", "Sy-Yen Kuo", "Tsung-Yi Ho" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "kernel-closed-form" ], "applications": [ "backdoor" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Counters the view that kernel-based dataset distillation counteracts backdoor risk, deriving two new theory-driven trigger-generation methods specialized for kernel-based distillation from a theoretical analysis of backdoor attacks under kernel methods. Reports its optimization-based trigger design produces resilient backdoor attacks that evade conventional backdoor detection and mitigation methods.", "abstract": "Dataset distillation offers a potential means to enhance data efficiency in deep learning. Recent studies have shown its ability to counteract backdoor risks present in original training samples. In this study, we delve into the theoretical aspects of backdoor attacks and dataset distillation based on kernel methods. We introduce two new theory-driven trigger pattern generation methods specialized for dataset distillation. Following a comprehensive set of analyses and experiments, we show that our optimization-based trigger design framework informs effective backdoor attacks on dataset distillation. Notably, datasets poisoned by our designed trigger prove resilient against conventional backdoor attack detection and mitigation methods. Our empirical results validate that the triggers developed using our approaches are proficient at executing resilient backdoor attacks.", "awesome_sections": [ "Applications / Privacy" ], "notes": "" }, { "id": "minimax-2024", "type": "paper", "title": "Efficient Dataset Distillation via Minimax Diffusion", "short": "Minimax Diffusion", "arxiv": "2311.15529", "url": null, "date": "2023-11-27", "venue": { "name": "CVPR 2024", "status": "conference" }, "authors": [ "Jianyang Gu", "Saeed Vahidian", "Vyacheslav Kungurtsev", "Haonan Wang", "Wei Jiang", "Yang You", "Yiran Chen" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "diffusion-guided-synthesis" ], "facets": { "labels": "hard", "prior": [ "diffusion" ], "regime": "generator-fine-tuning", "scale": [ "imagenet-1k" ] }, "tier": "core", "links": { "code": "https://github.com/vimar-gu/MinimaxDiffusion", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Fine-tunes a diffusion model with additional minimax criteria that push generated images toward representativeness and diversity, modeled theoretically as hierarchical diffusion control, replacing the sample-wise iterative optimization of prior distillation methods; under 100 IPC on ImageNet-Woof, needs under one-twentieth the distillation time of prior methods while achieving better performance.", "abstract": "Dataset distillation reduces the storage and computational consumption of training a network by generating a small surrogate dataset that encapsulates rich information of the original large-scale one. However, previous distillation methods heavily rely on the sample-wise iterative optimization scheme. As the images-per-class (IPC) setting or image resolution grows larger, the necessary computation will demand overwhelming time and resources. In this work, we intend to incorporate generative diffusion techniques for computing the surrogate dataset. Observing that key factors for constructing an effective surrogate dataset are representativeness and diversity, we design additional minimax criteria in the generative training to enhance these facets for the generated images of diffusion models. We present a theoretical model of the process as hierarchical diffusion control demonstrating the flexibility of the diffusion process to target these criteria without jeopardizing the faithfulness of the sample to the desired distribution. The proposed method achieves state-of-the-art validation performance while demanding much less computational resources. Under the 100-IPC setting on ImageWoof, our method requires less than one-twentieth the distillation time of previous methods, yet yields even better performance. Source code and generated data are available in https://github.com/vimar-gu/MinimaxDiffusion.", "awesome_sections": [ "Main / Generative Distillation / Diffusion" ], "explained": { "before": "Pixel-level (IDC-1) and embedding-level (GLaD) sample-wise iterative optimization methods scale their parameter space with IPC and resolution: IDC-1 takes over 90 GPU-hours to distill IPC=100 on ImageWoof (longer than training on ImageWoof itself), and its per-pixel modifications shrink as IPC grows, so gains over random images vanish or reverse at large IPC; GLaD needs prohibitive GPU memory as IPC grows. Separately, sampling directly from a pretrained diffusion model (DiT) or naively fine-tuning it (Difffit) had not been evaluated as a distillation method.", "problem": "The paper's own diagnostic (t-SNE, Fig. 3) shows raw DiT samples are representative of high-density regions of the real distribution but fail to cover its sparser regions (low diversity), while random real-image selection has the opposite problem (diverse but not representative) — so neither raw diffusion sampling nor coreset selection alone gives an effective surrogate set, and this gap becomes the paper's target instead of a matching loss to optimize against.", "idea": "Fine-tune a pretrained class-conditional diffusion model (DiT) with two extra minimax loss terms, one pulling each generated sample's embedding toward the least-similar real sample in a memory bank (representativeness) and one pushing it away from the most-similar previously-generated sample in another memory bank (diversity), so a single short fine-tuning run yields a generator whose *sampling* (no further per-sample optimization) gives both properties at once, at any IPC.", "method": "DiT (ImageNet-pretrained, latent diffusion with encoder $E$/decoder $D$) is fine-tuned with Difffit-style parameter-efficient fine-tuning using $\\mathcal{L}=\\mathcal{L}_{simple}+\\lambda_r \\mathcal{L}_r+\\lambda_d\\mathcal{L}_d$, where $\\mathcal{L}_{simple}$ is the standard diffusion noise- prediction loss, $\\mathcal{L}_r=\\arg\\max_\\theta \\min_{m}\\sigma(\\hat{\\mathbf{z}}_\\theta(\\mathbf{z}_t, \\mathbf{c}),\\mathbf{z}_m)$ pulls the predicted clean embedding toward the *least* similar real sample stored in a real-embedding memory bank $\\mathcal{M}$ (cosine similarity $\\sigma$), and $\\mathcal{L}_d= \\arg\\min_\\theta \\max_{d}\\sigma(\\hat{\\mathbf{z}}_\\theta(\\mathbf{z}_t,\\mathbf{c}),\\mathbf{z}_d)$ pushes it away from the *most* similar previously-predicted embedding stored in a second memory bank $\\mathcal{D}$ of adjacent-iteration generations. Fine-tuning uses $\\lambda_r{=}0.002$, $\\lambda_d{=}0.008$, batch size 8, 8 epochs, AdamW lr 1e-3, image size 256x256, single RTX 4090; a theoretical section frames the combined objective as an idealized tri-level optimization (Eq. 7) approximating a stochastic-control problem, arguing representativeness and diversity can both be satisfied without a strict trade-off. At deployment, the fine-tuned model is sampled (50 DDIM-style denoising steps) for the desired number of images per class, conditioned only on the class label — no test-time guidance and no further per-sample optimization.", "evidence": "ImageWoof, hard labels, ResNet-18/ResNetAP-10/ConvNet-6 (Table 1, all methods reproduced under the same protocol): at IPC=100, Ours 65.7%/64.5%/61.1% vs. second-best DiT baseline 58.9%/56.4%/53.4% (+5.5-8.1 points, the headline gain the abstract cites); at IPC=10, Ours 39.2%/39.2%/37.0% vs. DiT 34.7%/34.7%/34.2%; full-dataset upper bound ~86-89%. ImageNette/ImageIDC (Table 3, hard labels, ResNet-10): consistent smaller but positive margins over DiT and DM at all IPC 10/20/50. ImageNet-1K (Table 4, RDED's validation protocol, 224x224): IPC=10 Ours 44.3% vs. RDED 42.0% vs. SRe2L 21.3% vs. DiT baseline 39.6%; IPC=50 Ours 58.6% vs. RDED 56.5% vs. SRe2L 46.8% — a decoupled-synthesis (SRe2L/RDED) head-to-head under a protocol the paper adopts but does not fully restate (RDED's own protocol typically trains with teacher soft-label relabeling, so this comparison may not be a hard-label-only one, unlike the paper's own ImageWoof/Nette/IDC tables). Distribution-matching quality (Table 2, MMD to real features): Ours lowest (4.0 at IPC=50) vs. DM 4.8, DiT 5.4, IDC-1 6.7. Cost: under 1 hour to fine-tune+generate IPC=100 for a 10-class subset on a single RTX 4090, vs. IDC-1's >90 hours (the \"under one-twentieth\" claim), with GPU memory constant across IPC (unlike IDC-1/GLaD). Key ablation (Table 5, ResNet-10, ImageWoof/ImageIDC): naive representativeness alignment (Eq. 2, pulling to the distribution center rather than the minimax version) *degrades* accuracy at every IPC relative to plain Difffit fine-tuning (e.g. ImageWoof IPC=50: 47.1% vs. 51.0% baseline); the minimax diversity term alone gives the largest single gain (54.9%/68.4% at IPC=50 on Woof/IDC); combining minimax representativeness + minimax diversity (the full method) gives the best result (56.3%/69.6%), confirming the minimax formulation (not naive centroid-matching) is what makes the representativeness term useful at all.", "limitations": "The diffusion model (DiT) is pretrained on ImageNet-1K, i.e. matched to the target distribution for every experiment in the paper (ImageWoof/Nette/IDC/100/1K are all ImageNet subsets or ImageNet-1K itself); no experiment tests an off-target or non-ImageNet-pretrained generator, so none of the reported gain isolates generator-domain-mismatch robustness the way GLaD's ablation did for GANs; no CIFAR/MNIST or small-image experiments (the method targets exactly the ImageNet-scale, high-resolution regime where pixel-space methods struggle); the paper's own stated limitation is that it addresses only the classification setting and leaves other data domains to future work; fine-tuning still requires a class- conditional generator pretrained at the target's scale/resolution, so the \"under 1 hour\" distillation cost excludes the (much larger) upfront DiT-pretraining cost; the ImageNet-1K comparison follows RDED's protocol by reference without the paper independently confirming its own label regime there." } }, { "id": "dataset-distillation-in-latent-2023", "type": "paper", "title": "Dataset Distillation in Latent Space", "arxiv": "2311.15547", "url": null, "date": "2023-11-27", "venue": { "name": "arXiv 2023", "status": "preprint" }, "authors": [ "Yuxuan Duan", "Jianfu Zhang", "Liqing Zhang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "generative-latent-optimization" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Moves three mainstream dataset-distillation algorithms from pixel space into the latent space of a pretrained autoencoder, cutting time and memory cost and allowing more latent codes to be stored than pixel images at the same budget, which extends distillation to higher-resolution datasets and larger data ratios than the pixel-space versions reach.", "abstract": "Dataset distillation (DD) is a newly emerging research area aiming at alleviating the heavy computational load in training models on large datasets. It tries to distill a large dataset into a small and condensed one so that models trained on the distilled dataset can perform comparably with those trained on the full dataset when performing downstream tasks. Among the previous works in this area, there are three key problems that hinder the performance and availability of the existing DD methods: high time complexity, high space complexity, and low info-compactness. In this work, we simultaneously attempt to settle these three problems by moving the DD processes from conventionally used pixel space to latent space. Encoded by a pretrained generic autoencoder, latent codes in the latent space are naturally info-compact representations of the original images in much smaller sizes. After transferring three mainstream DD algorithms to latent space, we significantly reduce time and space consumption while achieving similar performance, allowing us to distill high-resolution datasets or target at greater data ratio that previous methods have failed. Besides, within the same storage budget, we can also quantitatively deliver more latent codes than pixel-level images, which further boosts the performance of our methods.", "awesome_sections": [ "Main / Better Optimization" ], "facets": { "parameterization": [ "generative-latent" ] } }, { "id": "fred-2023", "type": "paper", "title": "Frequency Domain-based Dataset Distillation", "short": "FreD", "arxiv": "2311.08819", "url": null, "date": "2023-11-15", "venue": { "name": "NeurIPS 2023", "status": "conference" }, "authors": [ "Donghyeok Shin", "Seungjae Shin", "Il-Chul Moon" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "frequency" ], "labels": "hard", "prior": [ "none" ], "regime": "single-level", "scale": [ "mnist-cifar", "tiny-imagenet", "imagenet-subsets", "large-scale-other" ] }, "tier": "notable", "links": { "code": "https://github.com/sdh0818/FreD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "builds_on": [ "idc-2022", "haba-2022", "mtt-2022" ], "explained": { "before": "idc-2022 downsamples images and upsamples with a static bilinear function, which the paper argues is \"highly vulnerable to instance-specific information loss\" because the spatial domain spreads a signal's important information roughly uniformly across every pixel; haba-2022 uses a parameterized (hallucinator) decoder, which requires a separate storage budget for the decoder network itself.", "problem": "Spatial-domain parameterization cannot cheaply identify which dimensions of an instance matter most -- empirically, spatial-domain pixels show a roughly uniform explained-variance ratio and a roughly uniform distillation-loss gradient magnitude across dimensions -- so any spatial compression (downsampling) must discard information indiscriminately rather than selectively.", "idea": "Transform each instance to the frequency domain first, where natural images have an \"energy compaction\" property (information concentrated in a few frequency components); this lets a class-wise binary mask select only the top-k frequency dimensions ranked by explained variance ratio (EVR) for optimization and storage, using a free, parameter-free, differentiable inverse transform (no learned decoder) to reconstruct an image-shaped instance for any downstream distillation loss.", "method": "A synthetic frequency memory $F=\\{(f^{(i)},y^{(i)})\\}$ stores $d$-dimensional frequency-domain vectors (one per stored instance), each initialized from the DCT of a randomly sampled real image. A class-wise binary mask $M^{(c)}\\in\\{0,1\\}^d$, computed once from the dataset's per-dimension EVR (no training required), selects the top-k dimensions to keep; the other $d-k$ entries are exactly 0 and are not counted against the budget. Training optimizes only the $k$ non-zero frequency coefficients per stored instance under any existing distillation loss $\\mathcal{L}_{DD}$ (default: trajectory matching, mtt-2022), applied to $\\tilde S=\\mathcal{F}^{-1}(M\\odot F)$. Because each instance now costs only $kn$ instances. Hard, fixed labels.", "evidence": "CIFAR-10, IPC=1 (#Params=30.72k, i.e. equal total parameter budget across all methods including pixel baselines): FreD 60.6% vs. second-best IDC 50.0% (+10.6pp, Table 1); SVHN, same budget: FreD 82.2% vs. HaBa 69.8% (+12.4pp). CIFAR-100, IPC=50: FreD 47.8% vs. IDC (not reported)/HaBa 47.0%. Orthogonality across objectives (Table 2, CIFAR-10): applying FreD's parameterization on top of DC/DM/TM losses each gives the best result among parameterizations, up to +10.8pp over the second-best combination and +10.6pp in cross-architecture generalization (AlexNet/VGG11/ResNet18). ImageNet-Subset (128x128), IPC=2 (#Params=983.04k, Table 3): FreD beats IDC and HaBa at equal parameter budget. Decoded-instance increment at equal budget (vs. IPC=1 baseline): IDC and HaBa both decode 5x more instances; FreD decodes 16x more at small budgets (IPC=1), tapering to 4x at large budgets (IPC=50), because larger per-instance budgets let it allocate more dimensions per instance rather than maximizing instance count. Ablation (Fig. 8a): EVR-based masking beats low-pass, band-stop, random, and amplitude-based masking criteria at equal k; high-pass masking is drastically worse (14.3% vs. ~60% at IPC=1). Ablation on frequency transform (Fig. 8b): DCT, DFT and DWT all improve over the spatial baseline, DCT best. Budget-allocation ablation (Fig. 9): smaller budgets favor more instances at lower per-instance k; larger budgets favor fewer instances with higher k.", "limitations": "States its own limitations (Appendix G): the frequency-domain approach is demonstrated only for 2D/3D grid data; connecting frequency transforms to non-grid domains such as text is non-trivial and may not be as effective. Performance is \"highly dependent on masking strategy\" -- EVR masking assumes task-relevant information concentrates in low/mid frequencies, which the paper notes may not hold for tasks where discriminative signal sits in high-frequency regions (e.g. fine medical-imaging detail, digital watermarking), and no data-driven alternative to EVR is evaluated empirically (only a gradient-based masking idea is proposed as future work). Storage accounting treats the $d-k$ zeroed dimensions as free (\"negligible small bytes\"), i.e. the binary mask's own storage cost is not counted against the budget, which is generous relative to a strict bit-accounting standard." }, "summary": "Parameterizes each synthetic instance in the frequency domain instead of pixel space, selecting a subset of frequency dimensions by explained variance so that a limited budget is spent only on the most informative components; theoretically and empirically more budget-efficient than prior parameterizations, and composes with existing distillation methods for consistent gains across benchmarks.", "abstract": "This paper presents FreD, a novel parameterization method for dataset distillation, which utilizes the frequency domain to distill a small-sized synthetic dataset from a large-sized original dataset. Unlike conventional approaches that focus on the spatial domain, FreD employs frequency-based transforms to optimize the frequency representations of each data instance. By leveraging the concentration of spatial domain information on specific frequency components, FreD intelligently selects a subset of frequency dimensions for optimization, leading to a significant reduction in the required budget for synthesizing an instance. Through the selection of frequency dimensions based on the explained variance, FreD demonstrates both theoretical and empirical evidence of its ability to operate efficiently within a limited budget, while better preserving the information of the original dataset compared to conventional parameterization methods. Furthermore, based on the orthogonal compatibility of FreD with existing methods, we confirm that FreD consistently improves the performances of existing distillation methods over the evaluation scenarios with different benchmark datasets. We release the code at https://github.com/sdh0818/FreD.", "awesome_sections": [ "Main / Distilled Dataset Parametrization" ] }, { "id": "rat-bptt-2024", "type": "paper", "title": "Embarassingly Simple Dataset Distillation", "short": "RaT-BPTT", "arxiv": "2311.07025", "url": null, "date": "2023-11-13", "venue": { "name": "ICLR 2024", "status": "conference" }, "authors": [ "Yunzhen Feng", "Ramakrishna Vedantam", "Julia Kempe" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "meta-learning-bptt", "optimization-recipes" ], "facets": { "parameterization": [ "pixels" ], "labels": "mixed", "prior": [ "random-networks" ], "regime": "bilevel", "scale": [ "tiny-imagenet" ] }, "tier": "notable", "links": { "code": "https://github.com/fengyzpku/Simple_Dataset_Distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Revisits backpropagation-through-time dataset distillation directly as a bi-level optimization problem and introduces Random Truncated BPTT, combining gradient truncation with a random window over the unrolled steps to stabilize gradients and cover long-term dependencies, setting new state-of-the-art results across standard benchmarks and revealing pronounced intercorrelation among the resulting distilled points that a boosting mechanism can exploit to build near-optimal subsets at multiple budgets.", "abstract": "Dataset distillation extracts a small set of synthetic training samples from a large dataset with the goal of achieving competitive performance on test data when trained on this sample. In this work, we tackle dataset distillation at its core by treating it directly as a bilevel optimization problem. Re-examining the foundational back-propagation through time method, we study the pronounced variance in the gradients, computational burden, and long-term dependencies. We introduce an improved method: Random Truncated Backpropagation Through Time (RaT-BPTT) to address them. RaT-BPTT incorporates a truncation coupled with a random window, effectively stabilizing the gradients and speeding up the optimization while covering long dependencies. This allows us to establish new state-of-the-art for a variety of standard dataset benchmarks. A deeper dive into the nature of distilled data unveils pronounced intercorrelation. In particular, subsets of distilled datasets tend to exhibit much worse performance than directly distilled smaller datasets of the same size. Leveraging RaT-BPTT, we devise a boosting mechanism that generates distilled datasets that contain subsets with near optimal performance across different data budgets.", "awesome_sections": [ "Main / Better Optimization" ], "explained": { "before": "Responds to dd-2018/BPTT's memory and instability problems, and to the family of inner-loop surrogates that were built to avoid unrolling entirely -- kip-2021 (NTK), rfad-2022 (NNGP/random features), frepo-2022 (empirical feature kernel), rcig-2023 (convexified implicit gradients) -- as well as to outer-loop surrogate objectives dc-2021/dsa-2021 (gradient matching) and mtt-2022/ftd-2023 (trajectory matching), arguing all of these either restrict the loss to MSE and favor wide networks, or use proxy metrics whose alignment with true test accuracy is unclear.", "problem": "Diagnoses exactly why plain BPTT underperforms: (1) large memory/compute from storing the whole unrolled trajectory, (2) unstable meta-gradients from long products of Hessian matrices over a highly non-convex inner problem (training a network from scratch), and (3) that naive truncated BPTT (T-BPTT), while stabilizing gradients, discards the early-training information that matters most (T-BPTT converges faster initially but plateaus below full BPTT).", "idea": "Combine truncation (only backpropagate through a short window of M steps, for stability and memory) with randomization (place that window at a uniformly random position along a longer N-step unroll, resampled every outer step) so the resulting gradient is an unbiased-in-position subsample of the full BPTT gradient that still covers the entire trajectory over many outer steps, rather than only ever seeing the same (early or late) window.", "method": "Optimizes the distilled set $\\mathcal{U}$ directly by gradient descent on pixels (and, for larger IPC/many-class settings, learned labels), with no kernel, surrogate loss, or precomputed expert trajectories. Each outer step: sample an unrolling length $N\\in[M,T]$, initialize $\\theta_0\\sim p(\\theta)$, take $N$ inner SGD/Adam steps on $\\mathcal{U}$, but only accumulate gradients for the last $M$ steps (i.e., start backpropagating once $n=N-M$), then update $\\mathcal{U}$ from the resulting meta-gradient $\\mathcal{G}_{RaT\\text{-}BPTT}$ -- a random M-length subsample of the terms in the full BPTT sum (Eq. 4). No model pool, no wide-network bias, no MSE restriction; works for any architecture and any differentiable loss, using the Higher library for efficient meta-gradient computation. A separate Boost-DD algorithm (Algorithm 2) builds a nested, budget-flexible dataset by distilling successive IPC-b blocks with a reduced (\"stale\") learning rate on previously distilled blocks.", "evidence": "Standard benchmarks (Table 1, narrow ConvNet): CIFAR-10 IPC 1/10/50 = 53.2/69.4/75.3%; CIFAR-100 IPC 1/10/50 = 35.3/47.5/50.6%; CUB-200 IPC 1/10 = 13.8/17.7%; Tiny-ImageNet IPC 1/10 = 20.1/24.4% -- state of the art on CIFAR-10, CIFAR-100 and CUB-200 at IPC 10/50, and statistically matching rcig-2023 (a kernel-closed-form method) across IPC 1/10/50 overall, without any inner-loop approximation. Combined with linear-basis parameterization (Deng and Russakovsky 2022): CIFAR-10 IPC1/10 improves from 53.2/69.4% to 68.2/72.8% (+1.6 points over LinBa alone). Wide-vs-narrow-network transfer (Sec. 4.1): RaT-BPTT trained on a narrow network transfers to wide-network evaluation nearly losslessly (e.g. CIFAR-100 IPC1: 35.3% narrow vs. 36.5% wide-transfer), whereas kernel-based methods evaluated on the narrow networks they were not tuned for drop sharply (RCIG CIFAR-100 IPC1: 39.3% wide-evaluated vs. 35.5% when re-evaluated narrow; FRePo Tiny-ImageNet IPC10: 25.4% wide vs. 22.4% narrow) -- the paper's central claim that RaT-BPTT \"generalizes gracefully\" in both directions while surrogate methods are architecture-width-dependent. Ablation on the truncation window (Sec 4.3, Fig. 6): random-uniform window placement beats both a fixed T-BPTT window and forward/backward-moving windows throughout training, isolating randomized window position (not just truncation) as the mechanism; a hardness-weighted (non-uniform) sampling can add a further 0.4 points on CIFAR-10 IPC10 but is dropped to keep the method simple. Intercorrelation finding: subsampling 5 images/class from a jointly distilled IPC50 set performs worse than 5 random real images per class for both KIP- and RaT-BPTT-distilled data, though RaT-BPTT degrades less than KIP; Boost-DD's strongly-boosted variant nearly matches jointly-distilled accuracy while yielding genuinely nested, budget-flexible subsets. No GPU-hour/wall-clock headline number is given, but the paper states RaT-BPTT's memory use \"often exceed[s] that of directly training the model\" because it still must store all intermediate parameters across the unrolled window.", "limitations": "States explicitly (Appendix A) that GPU memory consumption is substantial -- often exceeding direct training -- because RaT-BPTT still unrolls and backpropagates over multiple steps, and that larger models would need activation checkpointing to manage this, unlike the kernel-closed-form family's closed-form (unrolling-free) inner solve. Design choices (window size, unrolling length, uniform vs. hardness-weighted window sampling) are guided by empirical tuning rather than theory. A hardness-score analysis (Appendix D) shows that increasing IPC does not proportionally distill harder (\"forgettable\") examples -- easy, unforgettable examples are captured well at all IPC, but hard examples remain under-distilled regardless of budget, identifying a specific bottleneck to further scaling. The boosting fix for intercorrelation is only partial: boosted blocks still show inter-block correlation (an early block's subsampled accuracy differs from a later block's), and only the (accuracy-costly) strongly-boosted variant fully removes cross-budget degradation. Evaluated only on image-classification benchmarks up to Tiny-ImageNet scale; no ImageNet-1K result and no soft-label/teacher-relabeling scheme." } }, { "id": "seqmatch-2023", "type": "paper", "title": "Sequential Subset Matching for Dataset Distillation", "short": "SeqMatch", "arxiv": "2311.01570", "url": null, "date": "2023-11-02", "venue": { "name": "NeurIPS 2023", "status": "conference" }, "authors": [ "Jiawei Du", "Qin Shi", "Joey Tianyi Zhou" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "trajectory-matching", "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/shqii1j/seqmatch", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Argues that optimizing an entire synthetic dataset as one static, uniformly-updated entity couples its instances together and prevents later-training-epoch high-level features from being captured, and instead generates the synthetic instances sequentially so each new subset acquires knowledge the earlier subsets have not, improving over prior matching-based methods on SVHN, CIFAR-10/100 and Tiny-ImageNet.", "abstract": "Dataset distillation is a newly emerging task that synthesizes a small-size dataset used in training deep neural networks (DNNs) for reducing data storage and model training costs. The synthetic datasets are expected to capture the essence of the knowledge contained in real-world datasets such that the former yields a similar performance as the latter. Recent advancements in distillation methods have produced notable improvements in generating synthetic datasets. However, current state-of-the-art methods treat the entire synthetic dataset as a unified entity and optimize each synthetic instance equally. This static optimization approach may lead to performance degradation in dataset distillation. Specifically, we argue that static optimization can give rise to a coupling issue within the synthetic data, particularly when a larger amount of synthetic data is being optimized. This coupling issue, in turn, leads to the failure of the distilled dataset to extract the high-level features learned by the deep neural network (DNN) in the latter epochs. In this study, we propose a new dataset distillation strategy called Sequential Subset Matching (SeqMatch), which tackles this problem by adaptively optimizing the synthetic data to encourage sequential acquisition of knowledge during dataset distillation. Our analysis indicates that SeqMatch effectively addresses the coupling issue by sequentially generating the synthetic instances, thereby enhancing its performance significantly. Our proposed SeqMatch outperforms state-of-the-art methods in various datasets, including SVNH, CIFAR-10, CIFAR-100, and Tiny ImageNet. Our code is available at https://github.com/shqii1j/seqmatch.", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ] }, { "id": "dream-2023-2", "type": "paper", "title": "DREAM+: Efficient Dataset Distillation by Bidirectional Representative Matching", "short": "DREAM+", "arxiv": "2310.15052", "url": null, "date": "2023-10-23", "venue": { "name": "arXiv 2023", "status": "preprint" }, "authors": [ "Yanqing Liu", "Jianyang Gu", "Kai Wang", "Zheng Zhu", "Kaipeng Zhang", "Wei Jiang", "Yang You" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/lyq312318224/DREAM", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Extends DREAM's representative-sample selection to bidirectional matching across multiple dimensions of the distillation objective, plugging into gradient-, distribution- and trajectory-matching frameworks alike and cutting the number of distillation iterations needed by more than 15x without hurting accuracy.", "abstract": "Dataset distillation plays a crucial role in creating compact datasets with similar training performance compared with original large-scale ones. This is essential for addressing the challenges of data storage and training costs. Prevalent methods facilitate knowledge transfer by matching the gradients, embedding distributions, or training trajectories of synthetic images with those of the sampled original images. Although there are various matching objectives, currently the strategy for selecting original images is limited to naive random sampling. We argue that random sampling overlooks the evenness of the selected sample distribution, which may result in noisy or biased matching targets. Besides, the sample diversity is also not constrained by random sampling. Additionally, current methods predominantly focus on single-dimensional matching, where information is not fully utilized. To address these challenges, we propose a novel matching strategy called Dataset Distillation by Bidirectional REpresentAtive Matching (DREAM+), which selects representative original images for bidirectional matching. DREAM+ is applicable to a variety of mainstream dataset distillation frameworks and significantly reduces the number of distillation iterations by more than 15 times without affecting performance. Given sufficient training time, DREAM+ can further improve the performance and achieve state-of-the-art results. We have released the code at github.com/NUS-HPC-AI-Lab/DREAM+.", "awesome_sections": [ "Main / Better Optimization" ], "builds_on": [ "dream-2023" ] }, { "id": "yoco-2023", "type": "paper", "title": "You Only Condense Once: Two Rules for Pruning Condensed Datasets", "short": "YOCO", "arxiv": "2310.14019", "url": null, "date": "2023-10-21", "venue": { "name": "NeurIPS 2023", "status": "conference" }, "authors": [ "Yang He", "Lingao Xiao", "Joey Tianyi Zhou" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/he-y/you-only-condense-once", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Introduces two pruning rules — a Low LBPE (Loss-Based Prediction Error) Score and Balanced Construction — that shrink an already-condensed dataset to smaller sizes on demand without any extra condensation runs, for on-device settings with varying and limited compute; on CIFAR-10 at IPC-10, reports 6.98-8.89 point gains over condensation baselines and 6.31-23.92 point gains over pruning baselines.", "abstract": "Dataset condensation is a crucial tool for enhancing training efficiency by reducing the size of the training dataset, particularly in on-device scenarios. However, these scenarios have two significant challenges: 1) the varying computational resources available on the devices require a dataset size different from the pre-defined condensed dataset, and 2) the limited computational resources often preclude the possibility of conducting additional condensation processes. We introduce You Only Condense Once (YOCO) to overcome these limitations. On top of one condensed dataset, YOCO produces smaller condensed datasets with two embarrassingly simple dataset pruning rules: Low LBPE Score and Balanced Construction. YOCO offers two key advantages: 1) it can flexibly resize the dataset to fit varying computational constraints, and 2) it eliminates the need for extra condensation processes, which can be computationally prohibitive. Experiments validate our findings on networks including ConvNet, ResNet and DenseNet, and datasets including CIFAR-10, CIFAR-100 and ImageNet. For example, our YOCO surpassed various dataset condensation and dataset pruning methods on CIFAR-10 with ten Images Per Class (IPC), achieving 6.98-8.89% and 6.31-23.92% accuracy gains, respectively. The code is available at: https://github.com/he-y/you-only-condense-once.", "awesome_sections": [ "Main / Better Optimization" ] }, { "id": "mirage-2024", "type": "paper", "title": "Mirage: Model-Agnostic Graph Distillation for Graph Classification", "short": "Mirage", "arxiv": "2310.09486", "url": null, "date": "2023-10-14", "venue": { "name": "ICLR 2024", "status": "conference" }, "authors": [ "Mridul Gupta", "Sahil Manchanda", "Hariprasad Kodamana", "Sayan Ranu" ], "kind": "method", "settings": [ "graph" ], "families": [ "dataset-quantization" ], "tier": "notable", "links": { "code": "https://github.com/frigategnn/Mirage", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Mirage condenses a graph-classification dataset by decomposing every input graph into the multiset of computation trees a message-passing GNN would build on it, then compressing the (typically skewed) frequency distribution of these computation trees into a concise summary, rather than emulating gradient flows on the full dataset. Because it needs no full-dataset training and no target architecture during distillation, Mirage is unsupervised and architecture-agnostic, and the paper reports better generalization accuracy, compression and distillation efficiency than prior graph distillation baselines.", "abstract": "GNNs, like other deep learning models, are data and computation hungry. There is a pressing need to scale training of GNNs on large datasets to enable their usage on low-resource environments. Graph distillation is an effort in that direction with the aim to construct a smaller synthetic training set from the original training data without significantly compromising model performance. While initial efforts are promising, this work is motivated by two key observations: (1) Existing graph distillation algorithms themselves rely on training with the full dataset, which undermines the very premise of graph distillation. (2) The distillation process is specific to the target GNN architecture and hyper-parameters and thus not robust to changes in the modeling pipeline. We circumvent these limitations by designing a distillation algorithm called Mirage for graph classification. Mirage is built on the insight that a message-passing GNN decomposes the input graph into a multiset of computation trees. Furthermore, the frequency distribution of computation trees is often skewed in nature, enabling us to condense this data into a concise distilled summary. By compressing the computation data itself, as opposed to emulating gradient flows on the original training set-a prevalent approach to date-Mirage transforms into an unsupervised and architecture-agnostic distillation algorithm. Extensive benchmarking on real-world datasets underscores Mirage's superiority, showcasing enhanced generalization accuracy, data compression, and distillation efficiency when compared to state-of-the-art baselines.", "notes": "No P1-P4 matching objective is used; classified under dataset-quantization as the closest fit (frequency/coverage-based summarization with no bi-level optimization)." }, { "id": "gdem-2024", "type": "paper", "title": "Graph Distillation with Eigenbasis Matching", "short": "GDEM", "arxiv": "2310.09202", "url": null, "date": "2023-10-13", "venue": { "name": "ICML 2024", "status": "conference" }, "authors": [ "Yang Liu", "Deyu Bo", "Chuan Shi" ], "kind": "method", "settings": [ "graph" ], "families": [ "distribution-matching" ], "tier": "notable", "links": { "code": "https://github.com/liuyang-tian/GDEM", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Proposes GDEM for graph distillation, which aligns the eigenbasis and node features of the synthetic and real graphs directly and replicates the real graph's spectrum, instead of relying on GNN-derived gradients, representations or trajectories that bias the synthetic spectrum and tie the result to one GNN architecture; adds a discrimination constraint to balance effectiveness and generalization, reporting state-of-the-art results with strong cross-architecture generalization and a theoretical guarantee that the synthetic graphs are restricted spectral approximations of the real ones.", "abstract": "The increasing amount of graph data places requirements on the efficient training of graph neural networks (GNNs). The emerging graph distillation (GD) tackles this challenge by distilling a small synthetic graph to replace the real large graph, ensuring GNNs trained on real and synthetic graphs exhibit comparable performance. However, existing methods rely on GNN-related information as supervision, including gradients, representations, and trajectories, which have two limitations. First, GNNs can affect the spectrum (i.e., eigenvalues) of the real graph, causing spectrum bias in the synthetic graph. Second, the variety of GNN architectures leads to the creation of different synthetic graphs, requiring traversal to obtain optimal performance. To tackle these issues, we propose Graph Distillation with Eigenbasis Matching (GDEM), which aligns the eigenbasis and node features of real and synthetic graphs. Meanwhile, it directly replicates the spectrum of the real graph and thus prevents the influence of GNNs. Moreover, we design a discrimination constraint to balance the effectiveness and generalization of GDEM. Theoretically, the synthetic graphs distilled by GDEM are restricted spectral approximations of the real graphs. Extensive experiments demonstrate that GDEM outperforms state-of-the-art GD methods with powerful cross-architecture generalization ability and significant distillation efficiency. Our code is available at https://github.com/liuyang-tian/GDEM.", "awesome_sections": [ "Applications / Graph Neural Network" ], "notes": "" }, { "id": "does-graph-distillation-see-2023", "type": "paper", "title": "Does Graph Distillation See Like Vision Dataset Counterpart?", "short": "SGDD", "arxiv": "2310.09192", "url": null, "date": "2023-10-13", "venue": { "name": "NeurIPS 2023", "status": "conference" }, "authors": [ "Beining Yang", "Kai Wang", "Qingyun Sun", "Cheng Ji", "Xingcheng Fu", "Hao Tang", "Yang You", "Jianxin Li" ], "kind": "method", "settings": [ "graph" ], "families": [], "tier": "notable", "links": { "code": "https://github.com/RingBDStack/SGDD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Analyzes graph condensation methods in the spectral domain and finds that prior approaches induce a substantial Laplacian Energy Distribution (LED) shift between synthetic and real graphs, hurting cross-architecture generalization and tasks like anomaly detection and link prediction; proposes SGDD, which broadcasts the original graph's structural information into synthetic-graph generation to reduce this shift, reporting 98.6% of full-graph accuracy on YelpChi at a 1000x size reduction and 17.6-31.4% smaller LED shifts across nine datasets.", "abstract": "Training on large-scale graphs has achieved remarkable results in graph representation learning, but its cost and storage have attracted increasing concerns. Existing graph condensation methods primarily focus on optimizing the feature matrices of condensed graphs while overlooking the impact of the structure information from the original graphs. To investigate the impact of the structure information, we conduct analysis from the spectral domain and empirically identify substantial Laplacian Energy Distribution (LED) shifts in previous works. Such shifts lead to poor performance in cross-architecture generalization and specific tasks, including anomaly detection and link prediction. In this paper, we propose a novel Structure-broadcasting Graph Dataset Distillation (SGDD) scheme for broadcasting the original structure information to the generation of the synthetic one, which explicitly prevents overlooking the original structure information. Theoretically, the synthetic graphs by SGDD are expected to have smaller LED shifts than previous works, leading to superior performance in both cross-architecture settings and specific tasks. We validate the proposed SGDD across 9 datasets and achieve state-of-the-art results on all of them: for example, on the YelpChi dataset, our approach maintains 98.6% test accuracy of training on the original graph dataset with 1,000 times saving on the scale of the graph. Moreover, we empirically evaluate there exist 17.6% ~ 31.4% reductions in LED shift crossing 9 datasets. Extensive experiments and analysis verify the effectiveness and necessity of the proposed designs. The code is available in the GitHub repository: https://github.com/RingBDStack/SGDD.", "awesome_sections": [ "Applications / Graph Neural Network" ], "notes": "Abstract does not name the base matching objective SGDD builds on; family left empty." }, { "id": "hmn-2024", "type": "paper", "title": "Leveraging Hierarchical Feature Sharing for Efficient Dataset Condensation", "short": "HMN", "arxiv": "2310.07506", "url": null, "date": "2023-10-11", "venue": { "name": "ECCV 2024", "status": "conference" }, "authors": [ "Haizhong Zheng", "Jiachen Sun", "Shutong Wu", "Bhavya Kailkhura", "Zhuoqing Mao", "Chaowei Xiao", "Atul Prakash" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "factorized-bases" ] }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Stores condensed data in a three-tier Hierarchical Memory Network representing dataset-, class- and instance-level features, matching the hierarchical way real images share features within a classification system; the hierarchy also keeps images independent enough to allow instance-level pruning of redundant information, and it outperforms prior parameterization baselines on five public datasets.", "abstract": "Given a real-world dataset, data condensation (DC) aims to synthesize a small synthetic dataset that captures the knowledge of a natural dataset while being usable for training models with comparable accuracy. Recent works propose to enhance DC with data parameterization, which condenses data into very compact parameterized data containers instead of images. The intuition behind data parameterization is to encode shared features of images to avoid additional storage costs. In this paper, we recognize that images share common features in a hierarchical way due to the inherent hierarchical structure of the classification system, which is overlooked by current data parameterization methods. To better align DC with this hierarchical nature and encourage more efficient information sharing inside data containers, we propose a novel data parameterization architecture, Hierarchical Memory Network (HMN). HMN stores condensed data in a three-tier structure, representing the dataset-level, class-level, and instance-level features. Another helpful property of the hierarchical architecture is that HMN naturally ensures good independence among images despite achieving information sharing. This enables instance-level pruning for HMN to reduce redundant information, thereby further minimizing redundancy and enhancing performance. We evaluate HMN on five public datasets and show that our proposed method outperforms all baselines.", "awesome_sections": [ "Main / Distilled Dataset Parametrization" ] }, { "id": "progressive-dd-2024", "type": "paper", "title": "Data Distillation Can Be Like Vodka: Distilling More Times For Better Quality", "short": "Progressive DD", "arxiv": "2310.06982", "url": null, "date": "2023-10-10", "venue": { "name": "ICLR 2024", "status": "conference" }, "authors": [ "Xuxi Chen", "Yu Yang", "Zhangyang Wang", "Baharan Mirzasoleiman" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/VITA-Group/ProgressiveDD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Argues a single synthetic subset cannot represent the drastically different training dynamics of early versus late training, and instead synthesizes multiple small synthetic subsets in sequence, each conditioned on the ones before it, training on their cumulative union without extra training time; improves existing distillation methods by up to 4.3 points and is the first to scale distillation to considerably larger synthetic datasets.", "abstract": "Dataset distillation aims to minimize the time and memory needed for training deep networks on large datasets, by creating a small set of synthetic images that has a similar generalization performance to that of the full dataset. However, current dataset distillation techniques fall short, showing a notable performance gap when compared to training on the original data. In this work, we are the first to argue that using just one synthetic subset for distillation will not yield optimal generalization performance. This is because the training dynamics of deep networks drastically change during the training. Hence, multiple synthetic subsets are required to capture the training dynamics at different phases of training. To address this issue, we propose Progressive Dataset Distillation (PDD). PDD synthesizes multiple small sets of synthetic images, each conditioned on the previous sets, and trains the model on the cumulative union of these subsets without requiring additional training time. Our extensive experiments show that PDD can effectively improve the performance of existing dataset distillation methods by up to 4.3%. In addition, our method for the first time enable generating considerably larger synthetic datasets.", "awesome_sections": [ "Main / Better Optimization" ] }, { "id": "krr-st-2024", "type": "paper", "title": "Self-Supervised Dataset Distillation for Transfer Learning", "short": "KRR-ST", "arxiv": "2310.06511", "url": null, "date": "2023-10-10", "venue": { "name": "ICLR 2024", "status": "conference" }, "authors": [ "Dong Bok Lee", "Seanie Lee", "Joonho Ko", "Kenji Kawaguchi", "Juho Lee", "Sung Ju Hwang" ], "kind": "method", "settings": [ "self-supervised" ], "families": [ "kernel-closed-form" ], "builds_on": [ "kip-2021", "frepo-2022" ], "tier": "core", "links": { "code": "https://github.com/db-Lee/selfsup_dd", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "KRR-ST distills an unlabeled dataset for self-supervised pre-training by replacing the naive bilevel SSL objective, whose synthetic-sample gradient is biased by augmentation/masking randomness, with an inner objective that regresses learnable target feature representations (MSE, no randomness) and an outer objective that matches the resulting model's representations to a self-supervised target model trained on the full dataset. With the feature extractor fixed, the final linear head is obtained in closed form via kernel ridge regression, and the method is validated on several transfer-learning applications.", "abstract": "Dataset distillation methods have achieved remarkable success in distilling a large dataset into a small set of representative samples. However, they are not designed to produce a distilled dataset that can be effectively used for facilitating self-supervised pre-training. To this end, we propose a novel problem of distilling an unlabeled dataset into a set of small synthetic samples for efficient self-supervised learning (SSL). We first prove that a gradient of synthetic samples with respect to a SSL objective in naive bilevel optimization is \\textit{biased} due to the randomness originating from data augmentations or masking. To address this issue, we propose to minimize the mean squared error (MSE) between a model's representations of the synthetic examples and their corresponding learnable target feature representations for the inner objective, which does not introduce any randomness. Our primary motivation is that the model obtained by the proposed inner optimization can mimic the \\textit{self-supervised target model}. To achieve this, we also introduce the MSE between representations of the inner model and the self-supervised target model on the original full dataset for outer optimization. Lastly, assuming that a feature extractor is fixed, we only optimize a linear head on top of the feature extractor, which allows us to reduce the computational cost and obtain a closed-form solution of the head with kernel ridge regression. We empirically validate the effectiveness of our method on various applications involving transfer learning.", "notes": "", "explained": { "before": "Supervised dataset distillation methods (DSA, DM, MTT, and kernel-ridge-regression methods KIP and FRePo) optimize synthetic images against a labeled classification objective and are not designed to produce a dataset useful for self-supervised (label-free) pre-training; a naive fix would substitute a standard SSL loss (e.g. Barlow Twins, SimCLR) for the classification loss inside the same bi-level optimization used by those methods.", "problem": "The naive substitution is unstable in practice, and the paper proves why: SSL losses are stochastic functions of a randomly sampled data augmentation or input mask, and Theorem 1 shows that the gradient of the outer SSL loss with respect to the synthetic images, when the inner loop uses only a finite number of stochastic augmentation samples, is a biased estimator of the true gradient in bi-level optimization (unlike in standard non-bilevel SSL training, where the same stochastic gradient is unbiased) — because bias/covariance terms coupling the augmentation randomness to the inner optimization's implicit dependence on $X_s$ do not cancel.", "idea": "Remove the randomness from both the inner and outer objectives by replacing the SSL loss with a deterministic mean-squared-error regression onto learnable target feature vectors: distill both synthetic examples $X_s$ and paired learnable target representations $Y_s$, train the inner model to regress $Y_s$ from $X_s$ (no augmentation, no randomness), and match the resulting model's representations on the real data to those of a self-supervised teacher model pretrained on the full dataset with a real SSL objective.", "method": "Pretrain a target model $g_\\phi$ (ResNet-18) on the full unlabeled dataset $X_t$ with a real SSL objective (Barlow Twins) — the only place actual self-supervised training with augmentations happens. Then jointly optimize $(X_s, Y_s)$ by inner objective $\\mathcal{L}_{inner}=\\frac{1}{2}\\|Y_s-\\hat{g}_ \\theta(X_s)\\|_F^2$ and outer objective $\\min_{X_s,Y_s}\\frac{1}{2}\\|g_\\phi(X_t)-\\hat{g}_{\\theta^*(X_s, Y_s)}(X_t)\\|_F^2$. Following FRePo, $\\hat{g}_\\theta$ is split into a feature extractor $f_\\omega$ and linear head $h_W$; a pool of $l{=}10$ feature extractors is maintained and periodically reset, and for each meta-update the linear head is solved in closed form via kernel ridge regression, $h_{W^*}(v) = v^\\top f_\\omega(X_s)^\\top(K_{X_s,X_s}+\\lambda I_m)^{-1}Y_s$, avoiding the need to unroll the inner loop. Stored artifact: the synthetic images $X_s$ and their learned target representations $Y_s$ (no labels, no generator). At deployment, a fresh feature extractor and head are pretrained on $(X_s,Y_s)$ by MSE regression, then the head is discarded and the feature extractor is fine-tuned with a fresh task-specific head on the labeled target dataset.", "evidence": "Table 1 (CIFAR100→{CIFAR10, Aircraft, Cars, CUB2011, Dogs, Flowers}, ConvNet3, 1,000 synthetic images = 2% compression): KRR-ST beats all 8 baselines (Random, Kmeans, DSA, DM, MTT, KIP, FRePo, w/o pre) on every target, e.g. Cars 28.68% vs. best baseline KIP 23.12%, Flowers 67.88% vs. KIP 63.04%. Table 2 (TinyImageNet→6 targets, ConvNet4, 2,000 images = 2%): larger margins, e.g. Aircraft 58.83% vs. MTT 48.21%, Cars 49.26% vs. MTT 30.35%. Table 3 (ImageNet→7 targets, ConvNet4, 1,000 images ≈0.08%, FRePo the only feasible supervised baseline due to memory): Aircraft 57.17% vs. FRePo 39.03%, Cars 46.95% vs. FRePo 20.00%. Architecture-generalization (Fig. 3, TinyImageNet-distilled, evaluated on VGG11/AlexNet/MobileNet/ResNet10): KRR-ST wins in nearly every architecture x target cell (one exception: MobileNet on Aircraft). Target-data-free KD (Table 4, TinyImageNet-distilled surrogate, CIFAR10 teacher): KRR-ST beats all baselines across 5 student architectures (e.g. ConvNet4 58.31% vs. best baseline Kmeans 52.00%), and notably the supervised DD baselines (DSA/DM/MTT/FRePo) underperform even Random selection in this setting.", "limitations": "Only classification is used as the downstream fine-tuning task, despite the method being framed as general transfer learning; relies on a single SSL objective (Barlow Twins) for the target model, so it is unclear how sensitive results are to the choice of SSL algorithm; the theoretical bias result (Theorem 1) motivates the method but is not itself an empirical ablation of how much bias the naive bi-level SSL baseline actually suffers; distillation cost (160,000 meta-update iterations per dataset) is not reported in wall-clock or GPU-hours; largest scale tested is ImageNet at 64x64 resolution condensed to ~0.08%, well below full ImageNet-1K resolution/scale work in the image-classification literature." } }, { "id": "datm-2024", "type": "paper", "title": "Towards Lossless Dataset Distillation via Difficulty-Aligned Trajectory Matching", "short": "DATM", "arxiv": "2310.05773", "url": null, "date": "2023-10-09", "venue": { "name": "ICLR 2024", "status": "conference" }, "authors": [ "Ziyao Guo", "Kai Wang", "George Cazenavette", "Hui Li", "Kaipeng Zhang", "Yang You" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "trajectory-matching" ], "facets": { "parameterization": [ "pixels" ], "labels": "learned", "prior": [ "trained-experts" ], "regime": "single-level", "scale": [ "tiny-imagenet" ] }, "tier": "core", "links": { "code": "https://github.com/GzyAftermath/DATM", "project": "https://gzyaftermath.github.io/DATM/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Aligns the training-stage difficulty of the expert-trajectory segments matched during trajectory-matching distillation with the size of the synthetic set — early, easy-pattern segments for small budgets and late, hard-pattern segments for larger ones — letting trajectory matching keep improving as the synthetic set grows and reporting the first near-lossless dataset-distillation results.", "abstract": "The ultimate goal of Dataset Distillation is to synthesize a small synthetic dataset such that a model trained on this synthetic set will perform equally well as a model trained on the full, real dataset. Until now, no method of Dataset Distillation has reached this completely lossless goal, in part due to the fact that previous methods only remain effective when the total number of synthetic samples is extremely small. Since only so much information can be contained in such a small number of samples, it seems that to achieve truly loss dataset distillation, we must develop a distillation method that remains effective as the size of the synthetic dataset grows. In this work, we present such an algorithm and elucidate why existing methods fail to generate larger, high-quality synthetic sets. Current state-of-the-art methods rely on trajectory-matching, or optimizing the synthetic data to induce similar long-term training dynamics as the real data. We empirically find that the training stage of the trajectories we choose to match (i.e., early or late) greatly affects the effectiveness of the distilled dataset. Specifically, early trajectories (where the teacher network learns easy patterns) work well for a low-cardinality synthetic set since there are fewer examples wherein to distribute the necessary information. Conversely, late trajectories (where the teacher network learns hard patterns) provide better signals for larger synthetic sets since there are now enough samples to represent the necessary complex patterns. Based on our findings, we propose to align the difficulty of the generated patterns with the size of the synthetic dataset. In doing so, we successfully scale trajectory matching-based methods to larger synthetic datasets, achieving lossless dataset distillation for the very first time. Code and distilled datasets are available at https://gzyaftermath.github.io/DATM.", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ], "builds_on": [ "mtt-2022" ], "explained": { "before": "Responds to mtt-2022 and ftd-2023, which always match expert-trajectory segments at one fixed (typically early) training stage regardless of synthetic-set size, and shows this is why trajectory-matching accuracy stalls or falls below random selection as IPC grows (kip-2021 and rcig-2023's kernel-closed-form results, and tesla-2023's soft-label ImageNet-1K numbers, are also tabulated as the wider contemporary field).", "problem": "Trajectory-matching methods remain effective only at extremely small IPC; as the synthetic-set size grows they plateau or lose to random selection, so no prior method reaches \"lossless\" distillation (matching full-dataset accuracy). The paper diagnoses the cause as a fixed matching target: always matching early-epoch (\"easy pattern\") expert-trajectory segments, which is the right amount of information for a handful of images but starves a large synthetic set of the harder, later-training patterns it has room to represent.", "idea": "Align which segment of the cached expert trajectory is matched -- early/easy vs. late/hard -- with the size of the synthetic set being distilled: small IPC matches early segments, large IPC matches later segments. Combine this with learned (optimized) soft labels, stabilized by a sequential easy-then-hard generation schedule, since naively optimizing soft labels destabilizes low-IPC training.", "method": "Builds directly on mtt-2022's trajectory-matching loss (expert trajectories $\\theta^*$ cached from real-data training; student trained N steps on synthetic images from a sampled start point $\\theta^*_t$, loss = normalized distance to $\\theta^*_{t+M}$). The novel control is sampling $t$ from a shrinking window $\\mathcal{U}(\\{\\theta^*_{T^-},\\ldots,\\theta^*_T\\})$ whose upper bound $T$ starts small and grows toward a dataset-specific ceiling $T^+$ over the course of distillation (a \"sequential generation\" curriculum), rather than mtt-2022's window fixed at the start of training. Synthetic labels are optimized jointly as $\\hat y_i=\\mathrm{softmax}(L_i)$, initialized from a pretrained model's logits rather than one-hot vectors. What is stored: per-class synthetic images (pixels) plus a learned soft-label vector per image; expert trajectories remain a synthesis-time cache.", "evidence": "Table 5 (ConvNet, learned soft labels): CIFAR-10 IPC1/10/50/500/1000 = 46.9+/-0.5 / 66.8+/-0.2 / 76.1+/-0.3 / 83.5+/-0.2 / 85.5+/-0.4% vs. full-data 84.8+/-0.1% -- IPC1000 (20% of CIFAR-10) *exceeds* full-dataset accuracy, the paper's \"lossless\" claim; CIFAR-100 IPC1/10/50/100 = 27.9+/-0.2 / 47.2+/-0.4 / 55.0+/-0.2 / 57.5+/-0.2% vs. full-data 56.2+/-0.3% (IPC100 = 20% also exceeds full data); Tiny-ImageNet IPC1/10/50 = 17.1+/-0.3 / 31.1+/-0.3 / 39.7+/-0.3% vs. full-data 37.6+/-0.4% (IPC50 = 10% exceeds full data). At the same table's matched IPC, DATM beats MTT (e.g. CIFAR-100 IPC50 55.0 vs. 47.7), FTD (55.0 vs. 48.5), TESLA (55.0 vs. 47.9) and the kernel-closed-form field (RCIG CIFAR-100 IPC50 46.7). Cross-architecture (CIFAR-10 IPC50, ConvNet-distilled data evaluated unseen): DATM 55.03 (ConvNet) / 51.71 (ResNet18) / 45.38 (VGG) / 45.7+ (AlexNet) vs. FTD 48.90/46.65/43.24/42.20 and MTT 45.68/42.56/41.22/40.29 -- DATM transfers better than prior trajectory-matching methods at every unseen architecture, not just its training architecture. Cost: distillation cost does not scale linearly with IPC (IPC1000 costs only 1.05x IPC50 because late-trajectory matching needs fewer optimization changes per image); run on 4x NVIDIA A100 (80GB each); reducing synthetic steps N from 80 to 40 halves GPU memory at a cost of ~0.4-0.8 accuracy points. Ablation (Fig. 3a/Table, CIFAR-100 IPC50): neither component alone reaches the full method -- no soft label/no difficulty alignment 48.50%, +soft label only 50.79%, +difficulty alignment only 52.96%, both together 55.03% -- isolating that difficulty alignment and learned soft labels are complementary, additive mechanisms, neither sufficient alone.", "limitations": "States explicitly that \"lossless\" holds only for the distillation backbone network (ConvNet); when evaluated on other unseen architectures a performance drop remains, attributed to different-capacity models needing different amounts of training data. Optimizing soft labels destabilizes low-IPC distillation unless paired with the sequential easy-to-hard generation curriculum, adding a new schedule to tune. Distillation cost, while sub-linear in IPC, still requires 4x 80GB A100 GPUs and the same cached-expert-trajectory infrastructure as mtt-2022; the paper states scaling trajectory matching to larger (e.g. ImageNet-1K-scale) datasets remains hard due to this cost and is left to future work, so \"lossless\" is demonstrated only at CIFAR-10/100 and Tiny-ImageNet scale." } }, { "id": "can-pre-trained-models-2023", "type": "paper", "title": "Can pre-trained models assist in dataset distillation?", "arxiv": "2310.03295", "url": null, "date": "2023-10-05", "venue": { "name": "arXiv 2023", "status": "preprint" }, "authors": [ "Yao Lu", "Xuguang Chen", "Yuchen Zhang", "Jianyang Gu", "Tianle Zhang", "Yifan Zhang", "Xiaoniu Yang", "Qi Xuan", "Kai Wang", "Yang You" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [], "tier": "catalogued", "links": { "code": "https://github.com/yaolu-zjut/DDInterpreter", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Empirically studies which properties of a pretrained model used to guide dataset distillation matter, finding that greater model diversity helps, that sub-optimal (not fully converged) models can outperform well-trained ones, and that only a reasonable domain match — not an exact one — is needed, then uses these findings to improve cross-architecture generalization over baseline distillation methods.", "abstract": "Dataset Distillation (DD) is a prominent technique that encapsulates knowledge from a large-scale original dataset into a small synthetic dataset for efficient training. Meanwhile, Pre-trained Models (PTMs) function as knowledge repositories, containing extensive information from the original dataset. This naturally raises a question: Can PTMs effectively transfer knowledge to synthetic datasets, guiding DD accurately? To this end, we conduct preliminary experiments, confirming the contribution of PTMs to DD. Afterwards, we systematically study different options in PTMs, including initialization parameters, model architecture, training epoch and domain knowledge, revealing that: 1) Increasing model diversity enhances the performance of synthetic datasets; 2) Sub-optimal models can also assist in DD and outperform well-trained ones in certain cases; 3) Domain-specific PTMs are not mandatory for DD, but a reasonable domain match is crucial. Finally, by selecting optimal options, we significantly improve the cross-architecture generalization over baseline DD methods. We hope our work will facilitate researchers to develop better DD techniques. Our code is available at https://github.com/yaolu-zjut/DDInterpreter.", "awesome_sections": [ "Main / Better Optimization" ] }, { "id": "datadam-2023", "type": "paper", "title": "DataDAM: Efficient Dataset Distillation with Attention Matching", "short": "DataDAM", "arxiv": "2310.00093", "url": null, "date": "2023-09-29", "venue": { "name": "ICCV 2023", "status": "conference" }, "authors": [ "Ahmad Sajedi", "Samir Khaki", "Ehsan Amjadian", "Lucy Z. Liu", "Yuri A. Lawryshyn", "Konstantinos N. Plataniotis" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "tier": "notable", "links": { "code": "https://github.com/DataDistillation/DataDAM", "project": "https://datadistillation.github.io/DataDAM/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "parameterization": [ "pixels" ], "labels": "hard", "prior": [ "random-networks" ], "regime": "single-level", "scale": [ "mnist-cifar", "tiny-imagenet", "imagenet-1k" ] }, "explained": { "before": "dm-2023 matches only the class-wise mean feature (a single statistic) between real and synthetic data; cafe-2022's richer feature alignment can bias images and adds time cost; gradient- and trajectory-matching methods (dc-2021, dsa-2021, mtt-2022) need bi-level optimization or stored expert trajectories and can be memory-hungry.", "problem": "Mean-only feature matching under-constrains the spatial structure that actually discriminates classes, while methods that add richer supervision (CAFE) sacrifice the efficiency that makes distribution matching attractive in the first place.", "idea": "Match the spatial attention maps produced by multiple layers of randomly initialized networks (which encode where a network is \"looking\") in addition to a single top-layer MMD term, giving synthetic images multi-scale spatial supervision while remaining as cheap as plain distribution matching.", "method": "Optimizes synthetic pixel images. For several random ConvNets, computes spatial attention at every intermediate layer as the normalized sum of absolute feature-map values across channels; the Spatial Attention Matching (SAM) loss is the MSE between real and synthetic normalized attention maps at each layer, and a complementary MMD loss aligns the final-layer embeddings (loss = SAM + lambda*MMD). No inner-loop training of the encoders (random init only) and no classification loss.", "evidence": "CIFAR-10, 50 IPC: 67.0% vs. DM 63.0% (Table 1, +4.0pp). CIFAR-100, 50 IPC: 49.4% vs. DM 42.9% (+6.5pp). TinyImageNet, 50 IPC: 28.7% vs. DM 25.3%. ImageNet-1K (64x64), 50 IPC: 15.5% vs. DM 11.4% (+4.1pp, Table 2). Cost (Table 4, CIFAR-10, per-step): DataDAM 0.09-0.16s and 3.45-3.72GB vs. DC 0.16s/3.3-4.5GB, DSA 0.22s, and MTT 0.36s with OOM at IPC=50 -- almost two orders of magnitude faster than DC/DSA and comparable to DM's own cost. Ablation (Table 5, CIFAR-10, IPC=10): MMD-only 48.9%, SAM-only 49.8%, feature-map transfer without attention 47.2%, SAM+MMD combined 54.2% -- neither loss alone is sufficient.", "limitations": "Authors state the formulation is limited to convolutional networks and struggles to generalize to Vision Transformers. Like all fixed-ratio distillation methods, it must be re-optimized whenever the IPC (distillation ratio) changes rather than supporting a single reusable representation across budgets." }, "summary": "Matches spatial attention maps computed at multiple layers of randomly initialized networks between real and synthetic batches, instead of matching gradients or single-layer features, reporting up to 6.5-point gains on CIFAR-100 and 4.1-point gains on ImageNet-1K over prior methods.", "abstract": "Researchers have long tried to minimize training costs in deep learning while maintaining strong generalization across diverse datasets. Emerging research on dataset distillation aims to reduce training costs by creating a small synthetic set that contains the information of a larger real dataset and ultimately achieves test accuracy equivalent to a model trained on the whole dataset. Unfortunately, the synthetic data generated by previous methods are not guaranteed to distribute and discriminate as well as the original training data, and they incur significant computational costs. Despite promising results, there still exists a significant performance gap between models trained on condensed synthetic sets and those trained on the whole dataset. In this paper, we address these challenges using efficient Dataset Distillation with Attention Matching (DataDAM), achieving state-of-the-art performance while reducing training costs. Specifically, we learn synthetic images by matching the spatial attention maps of real and synthetic data generated by different layers within a family of randomly initialized neural networks. Our method outperforms the prior methods on several datasets, including CIFAR10/100, TinyImageNet, ImageNet-1K, and subsets of ImageNet-1K across most of the settings, and achieves improvements of up to 6.5% and 4.1% on CIFAR100 and ImageNet-1K, respectively. We also show that our high-quality distilled images have practical benefits for downstream applications, such as continual learning and neural architecture search.", "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ], "builds_on": [ "dm-2023" ] }, { "id": "cat-2023", "type": "paper", "title": "CaT: Balanced Continual Graph Learning with Graph Condensation", "short": "CaT", "arxiv": "2309.09455", "url": null, "date": "2023-09-18", "venue": { "name": "ICDM 2023", "status": "conference" }, "authors": [ "Yilun Liu", "Ruihong Qiu", "Zi Huang" ], "kind": "application", "settings": [ "graph" ], "families": [], "applications": [ "continual-learning" ], "tier": "catalogued", "links": { "code": "https://github.com/superallen13/CaT-CGL", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Proposes CaT for continual graph learning: before each model update, the newly arrived graph is condensed into a small synthetic replay graph stored in a Condensed Graph Memory alongside condensed replays of past graphs, and the model is updated by training directly on this memory (Training in Memory) rather than on the full incoming graph, which addresses the scale imbalance between new and historical data; reports superior effectiveness and efficiency over replay-based continual graph learning baselines on four benchmarks.", "abstract": "Continual graph learning (CGL) is purposed to continuously update a graph model with graph data being fed in a streaming manner. Since the model easily forgets previously learned knowledge when training with new-coming data, the catastrophic forgetting problem has been the major focus in CGL. Recent replay-based methods intend to solve this problem by updating the model using both (1) the entire new-coming data and (2) a sampling-based memory bank that stores replayed graphs to approximate the distribution of historical data. After updating the model, a new replayed graph sampled from the incoming graph will be added to the existing memory bank. Despite these methods are intuitive and effective for the CGL, two issues are identified in this paper. Firstly, most sampling-based methods struggle to fully capture the historical distribution when the storage budget is tight. Secondly, a significant data imbalance exists in terms of the scales of the complex new-coming graph data and the lightweight memory bank, resulting in unbalanced training. To solve these issues, a Condense and Train (CaT) framework is proposed in this paper. Prior to each model update, the new-coming graph is condensed to a small yet informative synthesised replayed graph, which is then stored in a Condensed Graph Memory with historical replay graphs. In the continual learning phase, a Training in Memory scheme is used to update the model directly with the Condensed Graph Memory rather than the whole new-coming graph, which alleviates the data imbalance problem. Extensive experiments conducted on four benchmark datasets successfully demonstrate superior performances of the proposed CaT framework in terms of effectiveness and efficiency. The code has been released on https://github.com/superallen13/CaT-CGL.", "awesome_sections": [ "Applications / Graph Neural Network" ], "notes": "Abstract does not name the condensation mechanism used inside CaT's condensation step; family left empty pending a full-text check." }, { "id": "multi-source-domain-adaptation-2024", "type": "paper", "title": "Multi-Source Domain Adaptation meets Dataset Distillation through Dataset Dictionary Learning", "arxiv": "2309.07666", "url": null, "date": "2023-09-14", "venue": { "name": "ICASSP 2024", "status": "conference" }, "authors": [ "Eduardo Fernandes Montesuma", "Fred Ngolè Mboula", "Antoine Souloumiac" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "applications": [ "domain-adaptation" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Poses multi-source domain adaptation and dataset distillation as one joint problem (MSDA-DD), combining Wasserstein Barycenter Transport and Dataset Dictionary Learning from the MSDA literature with the distribution-matching distillation objective. Reports state-of-the-art adaptation performance on four benchmarks with as little as one distilled sample per class.", "abstract": "In this paper, we consider the intersection of two problems in machine learning: Multi-Source Domain Adaptation (MSDA) and Dataset Distillation (DD). On the one hand, the first considers adapting multiple heterogeneous labeled source domains to an unlabeled target domain. On the other hand, the second attacks the problem of synthesizing a small summary containing all the information about the datasets. We thus consider a new problem called MSDA-DD. To solve it, we adapt previous works in the MSDA literature, such as Wasserstein Barycenter Transport and Dataset Dictionary Learning, as well as DD method Distribution Matching. We thoroughly experiment with this novel problem on four benchmarks (Caltech-Office 10, Tennessee-Eastman Process, Continuous Stirred Tank Reactor, and Case Western Reserve University), where we show that, even with as little as 1 sample per class, one achieves state-of-the-art adaptation performance.", "awesome_sections": [ "Applications / Domain Adaptation" ], "notes": "" }, { "id": "dataset-condensation-via-generative-2023", "type": "paper", "title": "Dataset Condensation via Generative Model", "arxiv": "2309.07698", "url": null, "date": "2023-09-14", "venue": { "name": "arXiv 2023", "status": "preprint" }, "authors": [ "David Junhao Zhang", "Heng Wang", "Chuhui Xue", "Rui Yan", "Wenqing Zhang", "Song Bai", "Mike Zheng Shou" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "generator-as-dataset" ], "facets": { "parameterization": [ "generator-weights" ], "regime": "generator-fine-tuning" }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Condenses a dataset into a trained generative model rather than pixels, so storage stays roughly constant as classes or resolution grow, with intra-class and inter-class losses that push same-class samples apart and separate class centers to increase diversity and discriminability; reports the first successful ImageNet-1K condensation using a generative-model artifact.", "abstract": "Dataset condensation aims to condense a large dataset with a lot of training samples into a small set. Previous methods usually condense the dataset into the pixels format. However, it suffers from slow optimization speed and large number of parameters to be optimized. When increasing image resolutions and classes, the number of learnable parameters grows accordingly, prohibiting condensation methods from scaling up to large datasets with diverse classes. Moreover, the relations among condensed samples have been neglected and hence the feature distribution of condensed samples is often not diverse. To solve these problems, we propose to condense the dataset into another format, a generative model. Such a novel format allows for the condensation of large datasets because the size of the generative model remains relatively stable as the number of classes or image resolution increases. Furthermore, an intra-class and an inter-class loss are proposed to model the relation of condensed samples. Intra-class loss aims to create more diverse samples for each class by pushing each sample away from the others of the same class. Meanwhile, inter-class loss increases the discriminability of samples by widening the gap between the centers of different classes. Extensive comparisons with state-of-the-art methods and our ablation studies confirm the effectiveness of our method and its individual component. To our best knowledge, we are the first to successfully conduct condensation on ImageNet-1k.", "awesome_sections": [ "Main / Generative Distillation / GAN" ], "notes": "Reclassified from generative-latent-optimization to generator-as-dataset: the abstract explicitly stores/trains a generative model as the condensed artifact rather than optimizing latents of a frozen generator; matches docs/taxonomy.md's generator-as-dataset formulation." }, { "id": "dq-2023", "type": "paper", "title": "Dataset Quantization", "short": "DQ", "arxiv": "2308.10524", "url": null, "date": "2023-08-21", "venue": { "name": "ICCV 2023", "status": "conference" }, "authors": [ "Daquan Zhou", "Kai Wang", "Jianyang Gu", "Xiangyu Peng", "Dongze Lian", "Yifan Zhang", "Yang You", "Jiashi Feng" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "dataset-quantization" ], "tier": "core", "links": { "code": "https://github.com/magic-research/Dataset_Quantization", "project": null }, "facets": { "labels": "hard", "regime": "selection", "prior": [ "pretrained-classifier" ], "scale": [ "imagenet-1k", "mnist-cifar" ] }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "Gradient/trajectory/distribution-matching DD (DC, DSA, IDC, CAFE, DM, MTT) synthesizes images by matching a metric computed through a specific network, which the paper argues biases the result toward that architecture; DM alone needs 28,000 GPU hours to compress 60% of ImageNet-1K, and DD methods saturate below full-dataset accuracy as the keep ratio grows. Prior coreset methods (GraphCut/submodular selection, Craig, GradMatch, Glister, k-Center, Herding) generalize across architectures but, the paper argues, select in a single one-shot pass that is biased toward high-density regions, losing diversity especially at low keep ratios (DC-2018 itself notes coresets trail DD in the low-data regime).", "problem": "Neither existing family is deployable at ImageNet scale with lossless compression: DD is architecture-biased and too expensive to run once, let alone re-run per keep ratio; one-shot coreset selection is cheap and architecture-agnostic but its selected samples cluster in high-density regions of feature space, losing diversity and underperforming at low keep ratios.", "idea": "Diagnose *why* one-shot submodular coreset selection loses diversity (a closed-form radius bound showing later selections are dominated by \"distance to the unselected remainder,\" not diversity, when the keep ratio is low) and fix it by selecting recursively into a sequence of non-overlapping bins instead of one coreset, then sampling uniformly across all bins so the final set spans the full range of representativeness-vs-diversity trade-offs the bins cover; drop the least informative image patches to cut storage further and reconstruct them at train time with a pretrained MAE.", "method": "Given a submodular gain $P(x_k)=\\sum_{p\\in S_1^{k-1}}\\|f(p){-}f(x_k)\\|_2^2 - \\sum_{p\\in D\\setminus S_1^{k-1}}\\|f(p){-}f(x_k)\\|_2^2$ (GraphCut-style, using a feature extractor $f$ from a classifier pretrained 10 epochs on the full data), the paper proves the next selected sample's feature norm is bounded by $(\\tfrac{2k}{M-2k})^2(R_1^{k-1})^2$, showing diversity is unguaranteed when $M\\gg k$ (low keep ratio) because the \"distance to the remaining data\" term dominates; a second selection round on the remainder has a smaller denominator ($M-K-2k$) and provably larger diversity bound. Dataset Quantization (DQ) exploits this by recursively carving the dataset into $N$ non-overlapping bins (default $N{=}10$) via the same submodular-gain rule applied to successive remainders, then uniformly samples a fraction $\\rho$ (the data keep ratio) from every bin so the final set $S^*$ spans bins of differing representativeness/diversity trade-off rather than only the most \"representative\" region. Independently, each kept image is divided into patches, a patch-importance score (GradCAM-based) is used to drop the least informative fraction $\\theta$ (default 25%), and at training time the dropped patches are reconstructed by a pretrained MAE decoder. In the loop: one pretrained feature-extractor classifier (ResNet-18/ViT-Base) for bin construction and one pretrained MAE for patch reconstruction at train time; no bi-level optimization, no per-architecture matching metric during selection.", "evidence": "ImageNet-1K, hard labels, lossless-compression headline (Fig. 5b, Sec. 4.3): 60% keep ratio trains ResNet-18 to within the full-dataset accuracy band (\"lossless\"); the same 60% pretrained ResNet-50 weights transfer to COCO detection at 39.0% vs. 39.2% mAP full-data and ADE20K segmentation at 42.3% vs. 42.5% mIoU full-data (Sec. 1, Fig. 6). Cross-architecture generalization, CIFAR-10, same ResNet-18-sourced compressed set, five student architectures (Table 2): at $\\rho{=}10\\%$, DM (distribution matching) averages 39.5% across R18/R50/ViT/Swin/ConvNeXt (dropping to 21.6% on ViT, a 59.4-point loss relative to its own 81.2% on R18-not-shown-here-cf-Fig2c) while DQ averages 69.4% (+29.9 over DM) at the same $\\rho$, with the DQ-DM gap narrowing but persisting up to 30% ($+35.5$ at $\\rho{=}30\\%$ is the largest gap reported). Compute cost (Table 4, ImageNet 60% keep ratio): DM needs 91 GPU-hours total across its per-ratio re-runs (7/14/29/41 hours at 10/20/40/60% plus implied earlier ratios) vs. DQ's 1 GPU-hour (bin creation only; sampling thereafter is free per ratio) — the paper's headline 388x figure (28,000 vs. 72 GPU-hours) is stated in the abstract/ introduction for a different, larger prior DM run. Scalability sweep (Fig. 4a-b, CIFAR-10, ResNet-18): at very low keep ratio (1%) GraphCut coreset selection is worst of the three; at high keep ratio DM saturates and ends about 5 points below random sampling; DQ is highest across the full range in both regimes. Language task (Alpaca instruction tuning, LLaMA-7B, Table 5): 20% of instructions selected by DQ match 100%-data performance on the BBH/DROP/MMLU/HumanEval average (27.1% vs. 27.7%); the abstract's stronger claim of comparable performance from 2% of the data applies specifically to the BBH and DROP subsets, not the full four-benchmark average.", "limitations": "States its own limitation directly: bins are still selected recursively from the whole dataset, which is extra computational effort relative to a true one-shot method, and lists designing a single-pass DQ as future work, alongside extending to video and generative (AIGC) tasks. Observed: dataset-distillation baselines (DM, DC, gradient/trajectory matching) are compared only on CIFAR-10 and not run on ImageNet-1K \"due to the extremely large computational cost\" — so the headline ImageNet lossless-compression claim is never benchmarked against any DD method at that scale, only against coreset selection and random sampling. The recursive bin construction and MAE patch-reconstruction depend on a classifier pretrained on the full dataset (for features) and a pretrained MAE (for reconstruction) respectively, both themselves full-data-trained priors whose cost is not included in the reported \"1 GPU-hour\" bin-creation figure. Ablations (bin number $N$, patch-drop ratio $\\theta$) are run only on CIFAR-10/ResNet-18, not re-verified at ImageNet-1K scale." }, "summary": "Partitions a dataset into non-overlapping bins by a diversity-driven criterion and samples from every bin, decoupling the compressed subset from any specific matching architecture; reports the first successful ImageNet-1K-scale, distillation-free compression at state-of-the-art ratios, and shows that 60% of ImageNet plus 20% of Alpaca instruction data trains vision and language models with negligible accuracy loss across classification, segmentation, detection and instruction tuning.", "abstract": "State-of-the-art deep neural networks are trained with large amounts (millions or even billions) of data. The expensive computation and memory costs make it difficult to train them on limited hardware resources, especially for recent popular large language models (LLM) and computer vision models (CV). Recent popular dataset distillation methods are thus developed, aiming to reduce the number of training samples via synthesizing small-scale datasets via gradient matching. However, as the gradient calculation is coupled with the specific network architecture, the synthesized dataset is biased and performs poorly when used for training unseen architectures. To address these limitations, we present dataset quantization (DQ), a new framework to compress large-scale datasets into small subsets which can be used for training any neural network architectures. Extensive experiments demonstrate that DQ is able to generate condensed small datasets for training unseen network architectures with state-of-the-art compression ratios for lossless model training. To the best of our knowledge, DQ is the first method that can successfully distill large-scale datasets such as ImageNet-1k with a state-of-the-art compression ratio. Notably, with 60% data from ImageNet and 20% data from Alpaca's instruction tuning data, the models can be trained with negligible or no performance drop for both vision tasks (including classification, semantic segmentation, and object detection) as well as language tasks (including instruction tuning tasks such as BBH and DROP).", "awesome_sections": [ "Main / Dataset Quantization" ] }, { "id": "mtt-vl-2024", "type": "paper", "title": "Vision-Language Dataset Distillation", "short": "MTT-VL", "arxiv": "2308.07545", "url": null, "date": "2023-08-15", "venue": { "name": "TMLR 2024", "status": "journal" }, "authors": [ "Xindi Wu", "Byron Zhang", "Zhiwei Deng", "Olga Russakovsky" ], "kind": "method", "settings": [ "vision-language" ], "families": [ "trajectory-matching" ], "facets": { "labels": "hard", "prior": [ "trained-experts" ], "regime": "bilevel", "parameterization": [ "pixels", "embedding-features" ], "scale": [ "flickr30k-coco" ] }, "tier": "landmark", "links": { "code": "https://github.com/princetonvisualai/multimodal_dataset_distillation", "project": "https://princetonvisualai.github.io/multimodal_dataset_distillation/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "This is the first vision-language dataset distillation method, extending trajectory matching to image-text pairs by jointly distilling them in a contrastive formulation (since there are no discrete classes to condition on) and using LoRA matching for efficient trajectory matching in large vision-language models. Against adapted vision-language coreset-selection baselines, the paper reports nearly doubling Flickr30K image-to-text recall@1 (5.6% to 9.9%) while using 100 distilled pairs versus 1000 selected ones.", "abstract": "Dataset distillation methods reduce large-scale datasets to smaller sets of synthetic data, preserving sufficient information to quickly train a new model from scratch. However, prior work on dataset distillation has focused exclusively on image classification datasets, whereas modern large-scale datasets are primarily vision-language datasets. In this work, we design the first vision-language dataset distillation method, building on the idea of trajectory matching. A key challenge is that vision-language datasets do not have a set of discrete classes. To overcome this, our proposed method jointly distills image-text pairs in a contrastive formulation. Further, we leverage Low-Rank Adaptation (LoRA) matching to enable more efficient and effective trajectory matching in complex modern vision-language models. Since there are no existing baselines, we compare our distillation approach with three adapted vision-language coreset selection methods. We demonstrate significant improvements on the challenging Flickr30K and COCO retrieval benchmarks: for example, on Flickr30K, the best coreset selection method selecting 1000 image-text pairs for training achieves only 5.6% image-to-text retrieval accuracy (i.e., recall@1); in contrast, our dataset distillation almost doubles that to 9.9% with just 100 training pairs, an order of magnitude fewer.", "explained": { "before": "Before this paper, dataset distillation (trajectory matching in particular, MTT) had only been applied to single-modal image classification, where a fixed set of classes gives synthesis a natural unit (match per-class statistics or trajectories). No prior method existed for paired image-text data.", "problem": "Vision-language datasets have no discrete classes to condition synthesis on, so the class-wise machinery of prior trajectory matching does not transfer, and no baseline for this setting existed at all (the paper builds its own adapted coreset baselines to compare against).", "idea": "Treat the whole image-text corpus as one unlabeled pool and jointly distill synthetic images and synthetic text embeddings so that a student contrastively trained on them for a few steps lands close to where an expert trained on real pairs would land after many more steps; efficiency at the scale of modern vision-language backbones is recovered by matching only low-rank LoRA updates of a frozen pretrained encoder rather than its full trajectory.", "method": "Images (pixels) and text (768-d BERT embeddings) are both directly optimized parameters. A pretrained NFNet (or ViT) image encoder plus a trainable linear projection, and a frozen pretrained BERT text encoder plus a trainable linear projection, produce embeddings trained with a bidirectional (image-to-text and text-to-image) contrastive loss. Expert trajectories are pairs of (image-tower, text-tower) parameter checkpoints obtained by training on real data for several epochs; the distillation loss matches the student's trajectory after $N$ synthetic-data steps against the expert's trajectory after $M$ real-data steps, normalized by the expert's total movement: $\\|\\hat\\theta_{t+N}-\\theta^*_{t+M}\\|_2^2/\\|\\theta^*_t-\\theta^*_{t+M}\\|_2^2$, summed over the image and text towers. For ViT backbones, LoRA matrices ($A\\in\\mathbb{R}^{d\\times r}$, $B\\in\\mathbb{R}^{r\\times d}$) are matched instead of full weights, cutting the trajectory's trainable parameter count by about 79% (86M to 18M on ViT-Base/16).", "evidence": "With an NFNet image encoder and frozen BERT text encoder, on Flickr30K at 100 pairs the method reports TR@1/5/10 = 9.9/28.3/39.1 and IR@1/5/10 = 4.7/15.7/24.6 (Table 6); at 1000 pairs, TR@1/5/10 = 13.3/34.8/45.9 and IR@1/5/10 = 7.9/24.1/33.8, versus the best coreset baseline (K-center, 1000 pairs) at TR@1 = 5.6, IR@1 = 4.4 (Table 1), and versus full-dataset training at TR@1 = 33.9, IR@1 = 27.3 (Table 3). On COCO at 100/1000 pairs, TR@1 = 2.5/6.8, IR@1 = 1.3/3.3 (Table 6). Expert trajectories take about 6-15 GPU-hours on an 8-GPU A6000 node (expert training itself: one RTX 3090, 10 epochs at roughly 40 min/epoch). The key ablation is co-distillation versus unimodal distillation at 100 pairs (Table 5): distilling only text reaches TR@1 = 1.3, distilling only images reaches TR@1 = 3.5, while joint co-distillation reaches TR@1 = 9.9 - the cross-modal contrastive coupling, not either modality alone, drives the gain. A separate ablation (Table 10) shows sensitivity to the text backbone: swapping frozen BERT for frozen CLIP text features raises both the full-data expert ceiling (TR@1 61.2 vs 33.9) and the 100-pair distilled result (TR@1 31.4 vs 9.9) by a similar margin, so much of the reported number is a property of the chosen encoder pair rather than the distillation method alone.", "limitations": "The paper explicitly invokes the \"no free lunch\" theorem: results depend heavily on which image/text backbone and optimizer are used during distillation, and cross-architecture transfer is weak (distilling on NFNet and evaluating on ViT or NF-ResNet50 at 100 pairs collapses TR@1 from 9.9 to 3.1-5.2, Table 4). Absolute recall remains far below full-dataset training even at 1000 pairs. Text is distilled only as continuous BERT embeddings, not as readable tokens. No coreset-selection literature existed for this setting, so the authors constructed their own adapted baselines, which likely understate what a stronger coreset method could achieve." }, "notes": "" }, { "id": "rethinking-data-distillation-do-2023", "type": "paper", "title": "Rethinking Data Distillation: Do Not Overlook Calibration", "arxiv": "2307.12463", "url": null, "date": "2023-07-24", "venue": { "name": "ICCV 2023", "status": "conference" }, "authors": [ "Dongyao Zhu", "Bowen Lei", "Jie Zhang", "Yanbo Fang", "Ruqi Zhang", "Yiqun Xie", "Dongkuan Xu" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "applications": [ "calibration" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Finds that networks trained on distilled data resist standard calibration methods (temperature scaling, mixup), tracing the failure to a more concentrated distribution of maximum logits and to loss of semantically meaningful but classification-irrelevant information in the distilled data; proposes Masked Temperature Scaling and Masked Distillation Training to correct for these effects while preserving distillation efficiency.", "abstract": "Neural networks trained on distilled data often produce over-confident output and require correction by calibration methods. Existing calibration methods such as temperature scaling and mixup work well for networks trained on original large-scale data. However, we find that these methods fail to calibrate networks trained on data distilled from large source datasets. In this paper, we show that distilled data lead to networks that are not calibratable due to (i) a more concentrated distribution of the maximum logits and (ii) the loss of information that is semantically meaningful but unrelated to classification tasks. To address this problem, we propose Masked Temperature Scaling (MTS) and Masked Distillation Training (MDT) which mitigate the limitations of distilled data and achieve better calibration results while maintaining the efficiency of dataset distillation.", "awesome_sections": [ "Applications / Robustness" ], "notes": "" }, { "id": "idm-2023", "type": "paper", "title": "Improved Distribution Matching for Dataset Condensation", "short": "IDM", "arxiv": "2307.09742", "url": null, "date": "2023-07-19", "venue": { "name": "CVPR 2023", "status": "conference" }, "authors": [ "Ganlong Zhao", "Guanbin Li", "Yipeng Qin", "Yizhou Yu" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "tier": "core", "links": { "code": "https://github.com/uitrbn/IDM", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "parameterization": [ "pixels" ], "labels": "hard", "prior": [ "random-networks" ], "regime": "single-level", "scale": [ "mnist-cifar", "tiny-imagenet", "imagenet-subsets" ] }, "explained": { "before": "dm-2023's plain distribution matching extracts a single feature (the class mean) per image and draws its embedding networks purely from random initialization.", "problem": "Two shortcomings of naive distribution matching: (1) imbalanced feature numbers -- comparing thousands of real features against only tens of synthetic ones makes the empirical MMD statistically unreliable unless the real data is highly redundant; (2) unvalidated embeddings -- purely randomly initialized networks occupy only a small, non-representative slice of the space of useful embedding functions, unlike the implicit sweep over trained networks that gradient-matching methods perform.", "idea": "Keep dm-2023's bi-level-free objective but fix its statistics: extract more synthetic features per image via a partition-and-expand augmentation, draw embeddings from a growing pool of partially trained (not just randomly initialized) networks, and add a class-aware regularizer that pulls synthetic features toward correct classification.", "method": "Each synthetic image is partitioned into l x l tiles that are differentiably upsampled back to full size, turning one image into l^2 feature-producing views for the matching loss (not extra stored data). A fixed-size FIFO \"model queue\" is periodically pushed with freshly initialized networks, trained K steps on real data, and popped when full, so embeddings are drawn from a mix of random-init through lightly trained networks rather than random init alone. An accuracy-weighted cross-entropy term (class-aware distribution regularization) is added on top of the same class-wise mean-matching loss as dm-2023. No bi-level loop; only the synthetic images are updated by gradient descent.", "evidence": "CIFAR-10, 10 IPC: IDM 58.6% vs. DM 48.9% (Table 1). CIFAR-100, 10 IPC: 45.1% vs. DM 29.7%, vs. DSA 32.3%. TinyImageNet, 10 IPC: 21.9% vs. DM 12.9%. ImageNet-Subset, 10 IPC: 17.1% vs. DM 11.9%. Cumulative ablation (Table 2, CIFAR-100, 10 IPC): DM 29.7% -> +model sampling 32.0% -> +class-aware regularization 34.3% -> +partition-and-expand augmentation 45.1% -- the augmentation gives by far the largest single gain (+10.8pp). On ImageNet-Subset, model sampling alone slightly hurts (-1.1pp), which the paper attributes to the model queue being undersized for a harder dataset. Experiments run on a single RTX 3090; no wall-clock or memory comparison to DM or to trajectory-matching methods is reported.", "limitations": "Introduces extra hyperparameters (queue size N_max, per-model training steps K, regularization weight, tuned differently per IPC) beyond DM's near-hyperparameter-free recipe. The model queue can underperform on harder/larger datasets (ImageNet-Subset). No explicit cost/time comparison is given against DM, bi-level gradient matching, or trajectory matching, so its efficiency advantage over those methods is argued qualitatively rather than measured in this source." }, "summary": "Identifies two weaknesses of plain distribution matching — an imbalanced number of features per matching term and embeddings not validated for the distance computation — and addresses them with partitioning-and-expansion augmentation, a richer pool of partially trained sampled models, and class-aware distribution regularization, outperforming most optimization-oriented distillation methods at a fraction of their compute.", "abstract": "Dataset Condensation aims to condense a large dataset into a smaller one while maintaining its ability to train a well-performing model, thus reducing the storage cost and training effort in deep learning applications. However, conventional dataset condensation methods are optimization-oriented and condense the dataset by performing gradient or parameter matching during model optimization, which is computationally intensive even on small datasets and models. In this paper, we propose a novel dataset condensation method based on distribution matching, which is more efficient and promising. Specifically, we identify two important shortcomings of naive distribution matching (i.e., imbalanced feature numbers and unvalidated embeddings for distance computation) and address them with three novel techniques (i.e., partitioning and expansion augmentation, efficient and enriched model sampling, and class-aware distribution regularization). Our simple yet effective method outperforms most previous optimization-oriented methods with much fewer computational resources, thereby scaling data condensation to larger datasets and models. Extensive experiments demonstrate the effectiveness of our method. Codes are available at https://github.com/uitrbn/IDM", "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ], "builds_on": [ "dm-2023" ] }, { "id": "towards-trustworthy-dataset-distillation-2024", "type": "paper", "title": "Towards Trustworthy Dataset Distillation", "short": "TrustDD", "arxiv": "2307.09165", "url": null, "date": "2023-07-18", "venue": { "name": "PR 2024", "status": "journal" }, "authors": [ "Shijie Ma", "Fei Zhu", "Zhen Cheng", "Xu-Yao Zhang" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [], "applications": [ "ood-detection" ], "tier": "catalogued", "links": { "code": "https://github.com/mashijie1028/TrustDD/", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Observes that dataset distillation has only ever targeted closed-world in-distribution classification, ignoring out-of-distribution detection, and proposes Trustworthy Dataset Distillation (TrustDD), which distills both in-distribution samples and outliers so the condensed set trains models competent at both tasks; Pseudo-Outlier Exposure corrupts in-distribution samples to stand in for real outlier data when none is available. Reports POE surpasses the state-of-the-art Outlier Exposure method and that TrustDD generalizes better to open-world scenarios than preceding DD.", "abstract": "Efficiency and trustworthiness are two eternal pursuits when applying deep learning in real-world applications. With regard to efficiency, dataset distillation (DD) endeavors to reduce training costs by distilling the large dataset into a tiny synthetic dataset. However, existing methods merely concentrate on in-distribution (InD) classification in a closed-world setting, disregarding out-of-distribution (OOD) samples. On the other hand, OOD detection aims to enhance models' trustworthiness, which is always inefficiently achieved in full-data settings. For the first time, we simultaneously consider both issues and propose a novel paradigm called Trustworthy Dataset Distillation (TrustDD). By distilling both InD samples and outliers, the condensed datasets are capable of training models competent in both InD classification and OOD detection. To alleviate the requirement of real outlier data, we further propose to corrupt InD samples to generate pseudo-outliers, namely Pseudo-Outlier Exposure (POE). Comprehensive experiments on various settings demonstrate the effectiveness of TrustDD, and POE surpasses the state-of-the-art method Outlier Exposure (OE). Compared with the preceding DD, TrustDD is more trustworthy and applicable to open-world scenarios. Our code is available at https://github.com/mashijie1028/TrustDD", "awesome_sections": [ "Applications / Robustness" ], "notes": "families left empty: TrustDD is a paradigm (distill InD + outliers) applicable atop any DD method, not one mechanism family." }, { "id": "sre2l-2023", "type": "paper", "title": "Squeeze, Recover and Relabel: Dataset Condensation at ImageNet Scale From A New Perspective", "short": "SRe2L", "arxiv": "2306.13092", "url": null, "date": "2023-06-22", "venue": { "name": "NeurIPS 2023", "status": "conference" }, "authors": [ "Zeyuan Yin", "Eric Xing", "Zhiqiang Shen" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "tier": "landmark", "links": { "code": "https://github.com/VILA-Lab/SRe2L/tree/main/SRe2L", "project": "https://zeyuanyin.github.io/projects/SRe2L/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "labels": "soft-relabel", "prior": [ "pretrained-classifier" ], "regime": "decoupled", "parameterization": [ "pixels" ], "scale": [ "imagenet-1k", "tiny-imagenet" ] }, "explained": { "before": "Names four prior categories it responds to: meta-model matching (DD, KIP, RFAD, FRePo, LinBa), gradient matching (DC, DSA, DCC, IDC), distribution matching (DM, CAFE, HaBa, IT-GAN, KFS) and trajectory matching (MTT, TESLA). All of these differentiate through, or otherwise couple to, an inner training loop on a model, which the paper identifies as the reason they stay confined to small or downsampled datasets.", "problem": "The bilevel optimization shared by prior paradigms unrolls many inner-loop model updates per outer-loop synthetic-data update, which is computationally prohibitive at ImageNet resolution and scale, and truncated unrolling introduces its own bias. The paper's diagnosis is that coupling the synthetic data to a specific model being trained on it, rather than the objective itself, is what blocks scaling.", "idea": "Split the pipeline into three independent, sequential, single-level stages: train a model once on the real data (squeeze), synthesize images by inverting that frozen model's batch-norm statistics and predictions with no real data present (recover), and train student models against the same model's soft labels on crops of the synthetic images (relabel). Because real and synthetic data are never co-resident and only summary statistics of the teacher are used, resolution and evaluation architecture become free choices.", "method": "Squeeze trains (or reuses off-the-shelf) a classifier $f_{\\theta_{\\mathcal T}}$ on the real data, e.g. torchvision ResNet-18/ResNet-50 for ImageNet-1K. Recover starts from Gaussian noise and updates pixels only (teacher weights frozen) to minimize classification loss against the target label plus a regularizer matching the synthetic batch's per-layer channel mean/variance to the teacher's stored BN running statistics: $\\ell(f_{\\theta_{\\mathcal T}}(\\tilde x),y)+\\alpha\\sum_l(\\|\\mu_l(\\tilde x)-\\mathrm{BN}_l^{RM}\\|_2+\\|\\sigma_l^2(\\tilde x)-\\mathrm{BN}_l^{RV}\\|_2)$, with RandomResizedCrop applied each iteration so only the cropped region is updated (found to matter more than any image-prior regularizer). Relabel stores FKD-style soft labels: the same teacher scores crops of the finished synthetic images, $\\tilde y_i=f_{\\theta_{\\mathcal T}}(\\tilde x_{R_i})$, and students are trained against these per-crop soft labels rather than the one-hot class label. Using a different architecture for recovery and relabeling is shown to hurt.", "evidence": "ImageNet-1K, ResNet-18 student, soft (FKD) labels, Table 4: IPC=10 21.3%, IPC=50 46.8%, IPC=100 52.8%, IPC=200 57.0%; ResNet-101 student reaches 60.8% at IPC=50. Tiny-ImageNet IPC=50, ResNet-18: 41.1%. No hard-label (one-hot) number is reported anywhere. Synthesis cost (Table 1, one RTX-4090, Tiny-ImageNet, ConvNet-4): 52x faster and 11.6x less memory than MTT; on ResNet-18, 16x faster and 6.4x less memory. Key ablation (Table 8): removing the TV/$\\ell_2$ image-prior regularizers and adding the per-iteration random crop lifts ImageNet-1K accuracy from 40.4% to 46.7% (ResNet-18, IPC=50) — the multi-crop optimization, not the pixel-prior terms, is the dominant lever. A separate ablation (Table 3) shows recovery budget matters: 0.5k/1k/2k/4k iterations give 38.9/43.7/46.7/46.7%.", "limitations": "States a persistent accuracy gap to full-data training and flags the extra storage needed for soft labels as a cost, without quantifying it. No results below IPC=10 and no ImageNet-21K results are reported (scaling to 21K and to other modalities is left as future work, later taken up by CDA). ViT/DeiT-Tiny students perform far worse than convolutional students on the same condensed data (15-33% vs 47-61%), and recovering from a larger or more heavily-trained teacher (ResNet-50 vs ResNet-18, or longer squeeze schedules) is shown to produce worse, not better, synthetic data — an architecture- and training-recipe-dependence that the paper surfaces but does not resolve." }, "builds_on": [], "summary": "Introduces Squeeze, Recover and Relabel (SRe2L), decoupling the bilevel optimization between model and synthetic data: a teacher is trained once on the real data (squeeze), synthetic images are then optimized by matching the frozen teacher's batch-norm statistics and predictions (recover), and student networks are trained against the teacher's soft labels (relabel); the pipeline that first made ImageNet-1K-scale condensation practical, reaching 60.8% top-1 accuracy on ImageNet-1K and 42.5% on Tiny-ImageNet at IPC=50, more than 30 points above prior methods, and 16-52x faster than MTT with far less memory.", "abstract": "We present a new dataset condensation framework termed Squeeze, Recover and Relabel (SRe$^2$L) that decouples the bilevel optimization of model and synthetic data during training, to handle varying scales of datasets, model architectures and image resolutions for efficient dataset condensation. The proposed method demonstrates flexibility across diverse dataset scales and exhibits multiple advantages in terms of arbitrary resolutions of synthesized images, low training cost and memory consumption with high-resolution synthesis, and the ability to scale up to arbitrary evaluation network architectures. Extensive experiments are conducted on Tiny-ImageNet and full ImageNet-1K datasets. Under 50 IPC, our approach achieves the highest 42.5% and 60.8% validation accuracy on Tiny-ImageNet and ImageNet-1K, outperforming all previous state-of-the-art methods by margins of 14.5% and 32.9%, respectively. Our approach also surpasses MTT in terms of speed by approximately 52$\\times$ (ConvNet-4) and 16$\\times$ (ResNet-18) faster with less memory consumption of 11.6$\\times$ and 6.4$\\times$ during data synthesis. Our code and condensed datasets of 50, 200 IPC with 4K recovery budget are available at https://github.com/VILA-Lab/SRe2L.", "awesome_sections": [ "Main / Decoupled Distillation" ] }, { "id": "sfgc-2023", "type": "paper", "title": "Structure-free Graph Condensation: From Large-scale Graphs to Condensed Graph-free Data", "short": "SFGC", "arxiv": "2306.02664", "url": null, "date": "2023-06-05", "venue": { "name": "NeurIPS 2023", "status": "conference" }, "authors": [ "Xin Zheng", "Miao Zhang", "Chunyang Chen", "Quoc Viet Hung Nguyen", "Xingquan Zhu", "Shirui Pan" ], "kind": "method", "settings": [ "graph" ], "families": [ "trajectory-matching", "kernel-closed-form" ], "facets": { "parameterization": [ "other" ], "labels": "hard", "prior": [ "trained-experts" ], "regime": "bilevel", "scale": [ "large-scale-other" ] }, "tier": "notable", "links": { "code": "https://github.com/amanda-zheng/sfgc", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "SFGC drops the joint node-and-structure optimization of prior graph condensation methods and instead distills only a condensed node set with topology reduced to the identity matrix (graph-free data), implicitly folding structural information into the node attributes; a training-trajectory meta-matching scheme aligns the long-term GNN learning behavior between the original and condensed graph, and a closed-form graph neural feature score dynamically evaluates candidate condensed data during training. The paper reports superior performance across different condensation ratios versus prior graph condensation baselines.", "abstract": "Graph condensation, which reduces the size of a large-scale graph by synthesizing a small-scale condensed graph as its substitution, has immediate benefits for various graph learning tasks. However, existing graph condensation methods rely on the joint optimization of nodes and structures in the condensed graph, and overlook critical issues in effectiveness and generalization ability. In this paper, we advocate a new Structure-Free Graph Condensation paradigm, named SFGC, to distill a large-scale graph into a small-scale graph node set without explicit graph structures, i.e., graph-free data. Our idea is to implicitly encode topology structure information into the node attributes in the synthesized graph-free data, whose topology is reduced to an identity matrix. Specifically, SFGC contains two collaborative components: (1) a training trajectory meta-matching scheme for effectively synthesizing small-scale graph-free data; (2) a graph neural feature score metric for dynamically evaluating the quality of the condensed data. Through training trajectory meta-matching, SFGC aligns the long-term GNN learning behaviors between the large-scale graph and the condensed small-scale graph-free data, ensuring comprehensive and compact transfer of informative knowledge to the graph-free data. Afterward, the underlying condensed graph-free data would be dynamically evaluated with the graph neural feature score, which is a closed-form metric for ensuring the excellent expressiveness of the condensed graph-free data. Extensive experiments verify the superiority of SFGC across different condensation ratios.", "builds_on": [ "gcond-2022" ], "notes": "", "explained": { "before": "GCond (and its one-step cousin DosCond) solve graph condensation by jointly optimizing condensed features, a learned/generated adjacency matrix, and GNN parameters — a triple-level problem — using short-range (single- or few-step) online gradient matching against the real graph's GNN gradients. SFGC argues this inherits two problems from that design rather than from graphs per se.", "problem": "Short-range online gradient matching only imitates a few steps of GNN training at a time (\"short-sight\"), so the condensed graph fails to capture the GNN's holistic, long-term learning behavior; and jointly learning node features, a graph-structure-learning module, and GNN parameters (triple-level) makes it hard to reach a jointly optimal solution and causes the condensed structure to overfit the specific GNN convolution used during condensation, hurting cross-architecture generalization.", "idea": "If the graph condensation goal is really about matching long-term GNN training behavior, drop the graph structure entirely — represent the condensed data as a bare node set with identity adjacency — and let the node features implicitly absorb whatever structural information matters, learned by matching whole segments of pretrained expert training trajectories (not single gradient steps) instead of raw gradients.", "method": "Offline, trains $K$ expert GNNs on the full graph and stores their parameter snapshots $\\{\\theta_t^{*,i}\\}_{t=1}^T$ as trajectories. Online, samples a trajectory and a start step $t_0$, initializes a student GNN$_\\mathcal{S}$ at $\\theta_{t_0}^{*,i}$, trains it for $q$ steps on the condensed graph-free data $\\mathcal{S}=(\\widetilde{\\mathbf{X}},\\mathbf{I},\\widetilde{\\mathbf{Y}})$ (identity adjacency, fixed hard labels), and minimizes a meta-matching loss $\\mathcal{L}_{\\text{meta-tt}}=\\|\\widetilde{\\theta}_{t_0+q}-\\theta_{t_0+p}^{*,i}\\|^2 / \\|\\widetilde{\\theta}_{t_0}-\\theta_{t_0+p}^{*,i}\\|^2$ that aligns the student's endpoint after $q$ steps to the expert's endpoint after $p$ steps, backpropagating through the $q$-step training to update $\\widetilde{\\mathbf{X}}$. Because a condensed-graph-free candidate cannot be scored without training a GNN on it, a graph neural tangent kernel (GNTK) ridge-regression \"graph neural feature score\" gives a closed-form validation error (a KIP-style criterion adapted to GNTK) used to pick the best candidate across the meta-matching trajectory without extra iterative training.", "evidence": "Table 1 (GCN condensation and evaluation, hard labels, node classification, 10 repeats): SFGC wins 13 of 15 dataset x ratio combinations against GCond, GCond-X, DC-Graph, coreset and coarsening baselines; on Cora r=2.6% SFGC reaches 81.7% vs. 81.2% whole-dataset (exceeding the original graph); on ogbn-arxiv r=0.05% SFGC reaches 65.5% vs. GCond's 61.3% (whole 71.4%), the largest margin over GCond in the table. Table 2 (cross-architecture transfer to MLP/GAT/APPNP/Cheby/GCN/SAGE/SGC): SFGC's average score beats GCond and GCond-X on both Citeseer (71.5 vs. 69.0/70.2) and Cora (80.3 vs. 78.4/75.7), including on GAT and MLP where GCond loses much more accuracy (e.g. GCond's GAT drops to 55.4/66.2 on Citeseer/Cora vs. SFGC's 72.1/80.8) — attributed directly to identity-structure data having nothing for any specific convolution operator to overfit. Key ablation (Fig. 3): five variants that add back a learned structure (discrete kNN with k=1/2/5, continuous cosine-similarity structure, a GCond-style parameterized structure module) all underperform the structure-free SFGC across three transductive datasets and nine ratios, and the parameterized-structure variant \"almost fails\" under the meta-matching objective. Efficiency (Appendix E.1): the GNTK-based score takes 0.015s vs. 0.845s for iterative-training evaluation on Flickr (r=0.1%), and 0.042s vs. 4.264s on ogbn-arxiv (r=0.05%).", "limitations": "Only node-level condensation of a single graph is addressed; the conclusion explicitly leaves graph-level condensation (condensing a set of many graphs, DosCond/SFGC's counterpart problem) to future work. The offline expert-trajectory pretraining is an added upfront cost not folded into the reported per-step efficiency numbers. Results are reported only with GCN as the trajectory-generating architecture (cross-architecture generalization is evaluated only at test time, not for experts of other architectures). The parameterized-structure ablation \"almost fails,\" suggesting the meta-matching objective is specifically tuned to the identity-structure case and may not transfer cleanly to methods that still want some structure." } }, { "id": "towards-efficient-deep-hashing-2023", "type": "paper", "title": "Towards Efficient Deep Hashing Retrieval: Condensing Your Data via Feature-Embedding Matching", "short": "IEM", "arxiv": "2305.18076", "url": null, "date": "2023-05-29", "venue": { "name": "arXiv 2023", "status": "preprint" }, "authors": [ "Tao Feng", "Jie Zhang", "Huashan Liu", "Zhijie Wang", "Shengyuan Pang" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "applications": [ "retrieval" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Adapts dataset condensation to deep hashing retrieval with IEM (Information-intensive feature Embedding Matching), a distribution-matching-centered method that adds model and data augmentation to strengthen the condensed hashing-space features, since retrieval training does not benefit directly from condensation methods designed for classification accuracy. Reports superior performance and efficiency relative to applying mainstream condensation methods to deep hashing retrieval.", "abstract": "Deep hashing retrieval has gained widespread use in big data retrieval due to its robust feature extraction and efficient hashing process. However, training advanced deep hashing models has become more expensive due to complex optimizations and large datasets. Coreset selection and Dataset Condensation lower overall training costs by reducing the volume of training data without significantly compromising model accuracy for classification task. In this paper, we explore the effect of mainstream dataset condensation methods for deep hashing retrieval and propose IEM (Information-intensive feature Embedding Matching), which is centered on distribution matching and incorporates model and data augmentation techniques to further enhance the feature of hashing space. Extensive experiments demonstrate the superior performance and efficiency of our approach.", "awesome_sections": [ "Applications / Retrieval" ], "notes": "" }, { "id": "gold-from-ores-2024", "type": "paper", "title": "Distill Gold from Massive Ores: Bi-level Data Pruning towards Efficient Dataset Distillation", "short": "Gold from Ores", "arxiv": "2305.18381", "url": null, "date": "2023-05-28", "venue": { "name": "ECCV 2024", "status": "conference" }, "authors": [ "Yue Xu", "Yong-Lu Li", "Kaitong Cui", "Ziyu Wang", "Cewu Lu", "Yu-Wing Tai", "Chi-Keung Tang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/silicx/GoldFromOres", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Studies which real training samples are essential to the distillation process itself, proposing a data-pruning criterion based on empirical loss value refined by each sample's causal effect on the distillation outcome, which improves several distillation algorithms' results, including on full ImageNet-1K and Kinetics-400.", "abstract": "Data-efficient learning has garnered significant attention, especially given the current trend of large multi-modal models. Recently, dataset distillation has become an effective approach by synthesizing data samples that are essential for network training. However, it remains to be explored which samples are essential for the dataset distillation process itself. In this work, we study the data efficiency and selection for the dataset distillation task. By re-formulating the dynamics of distillation, we provide insight into the inherent redundancy in the real dataset, both theoretically and empirically. We propose to use the empirical loss value as a static data pruning criterion. To further compensate for the variation of the data value in training, we find the most contributing samples based on their causal effects on the distillation. The proposed selection strategy can efficiently exploit the training dataset, outperform the previous SOTA distillation algorithms, and consistently enhance the distillation algorithms, even on much larger-scale and more heterogeneous datasets, e.g., full ImageNet-1K and Kinetics-400. We believe this paradigm will open up new avenues in the dynamics of distillation and pave the way for efficient dataset distillation. Our code is available on https://github.com/silicx/GoldFromOres-BiLP.", "awesome_sections": [ "Main / Better Optimization" ] }, { "id": "summarizing-stream-data-for-2024", "type": "paper", "title": "Summarizing Stream Data for Memory-Constrained Online Continual Learning", "short": "SSD", "arxiv": "2305.16645", "url": null, "date": "2023-05-26", "venue": { "name": "AAAI 2024", "status": "conference" }, "authors": [ "Jianyang Gu", "Kai Wang", "Wei Jiang", "Yang You" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "gradient-matching" ], "applications": [ "continual-learning" ], "tier": "notable", "links": { "code": "https://github.com/vimar-gu/SSD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "SSD summarizes the incoming data stream into more informative replay samples by distilling the training characteristics of real images, maintaining consistency of training gradients and relationship to past tasks so summarized samples represent the stream better than raw exemplars at the same buffer size. Reports more than a 3% accuracy boost on sequential CIFAR-100 under an extremely restricted memory buffer, with limited extra computational overhead.", "abstract": "Replay-based methods have proved their effectiveness on online continual learning by rehearsing past samples from an auxiliary memory. With many efforts made on improving training schemes based on the memory, however, the information carried by each sample in the memory remains under-investigated. Under circumstances with restricted storage space, the informativeness of the memory becomes critical for effective replay. Although some works design specific strategies to select representative samples, by only employing a small number of original images, the storage space is still not well utilized. To this end, we propose to Summarize the knowledge from the Stream Data (SSD) into more informative samples by distilling the training characteristics of real images. Through maintaining the consistency of training gradients and relationship to the past tasks, the summarized samples are more representative for the stream data compared to the original images. Extensive experiments are conducted on multiple online continual learning benchmarks to support that the proposed SSD method significantly enhances the replay effects. We demonstrate that with limited extra computational overhead, SSD provides more than 3% accuracy boost for sequential CIFAR-100 under extremely restricted memory buffer. Code in https://github.com/vimar-gu/SSD.", "awesome_sections": [ "Applications / Continual Learning" ], "notes": "" }, { "id": "size-approx-error-2023", "type": "paper", "title": "On the Size and Approximation Error of Distilled Sets", "short": "Size & approximation error", "arxiv": "2305.14113", "url": null, "date": "2023-05-23", "venue": { "name": "NeurIPS 2023", "status": "conference" }, "authors": [ "Alaa Maalouf", "Murad Tukan", "Noel Loo", "Ramin Hasani", "Mathias Lechner", "Daniela Rus" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [ "kernel-closed-form" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Gives the first proof, for kernel-ridge-regression-based distillation methods such as Kernel Inducing Points, that a small distilled set exists for shift-invariant kernels by transforming the ridge-regression problem into random Fourier feature space, showing the distilled-set size can be linear in the RFF-space dimension (or near-linear in the kernel's effective degrees of freedom) with an excess-risk bound that depends on the regularization parameter.", "abstract": "Dataset Distillation is the task of synthesizing small datasets from large ones while still retaining comparable predictive accuracy to the original uncompressed dataset. Despite significant empirical progress in recent years, there is little understanding of the theoretical limitations/guarantees of dataset distillation, specifically, what excess risk is achieved by distillation compared to the original dataset, and how large are distilled datasets? In this work, we take a theoretical view on kernel ridge regression (KRR) based methods of dataset distillation such as Kernel Inducing Points. By transforming ridge regression in random Fourier features (RFF) space, we provide the first proof of the existence of small (size) distilled datasets and their corresponding excess risk for shift-invariant kernels. We prove that a small set of instances exists in the original input space such that its solution in the RFF space coincides with the solution of the original data. We further show that a KRR solution can be generated using this distilled set of instances which gives an approximation towards the KRR solution optimized on the full input data. The size of this set is linear in the dimension of the RFF space of the input set or alternatively near linear in the number of effective degrees of freedom, which is a function of the kernel, number of datapoints, and the regularization parameter $\\lambda$. The error bound of this distilled set is also a function of $\\lambda$. We verify our bounds analytically and empirically.", "awesome_sections": [ "Main / Better Understanding" ] }, { "id": "a-comprehensive-study-on-2023", "type": "paper", "title": "A Comprehensive Study on Dataset Distillation: Performance, Privacy, Robustness and Fairness", "arxiv": "2305.03355", "url": null, "date": "2023-05-05", "venue": { "name": "arXiv 2023", "status": "preprint" }, "authors": [ "Zongxiong Chen", "Jiahui Geng", "Derui Zhu", "Herbert Woisetschlaeger", "Qing Li", "Sonja Schimmler", "Ruben Mayer", "Chunming Rong" ], "kind": "evaluation-benchmark", "settings": [ "image-classification" ], "families": [], "applications": [ "privacy", "robustness", "fairness" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Runs membership-inference, robustness, and per-class fairness evaluations across several dataset-distillation methods and reports that distillation reduces but does not eliminate privacy leakage, can degrade adversarial robustness to varying degrees, and can amplify unfairness across classes; proposes a large-scale benchmarking framework covering these axes.", "abstract": "The aim of dataset distillation is to encode the rich features of an original dataset into a tiny dataset. It is a promising approach to accelerate neural network training and related studies. Different approaches have been proposed to improve the informativeness and generalization performance of distilled images. However, no work has comprehensively analyzed this technique from a security perspective and there is a lack of systematic understanding of potential risks. In this work, we conduct extensive experiments to evaluate current state-of-the-art dataset distillation methods. We successfully use membership inference attacks to show that privacy risks still remain. Our work also demonstrates that dataset distillation can cause varying degrees of impact on model robustness and amplify model unfairness across classes when making predictions. This work offers a large-scale benchmarking framework for dataset distillation evaluation.", "awesome_sections": [ "Main / Benchmark" ], "notes": "" }, { "id": "survey-geng-2023", "type": "paper", "title": "A Survey on Dataset Distillation: Approaches, Applications and Future Directions", "short": "Geng et al. survey", "arxiv": "2305.01975", "url": null, "date": "2023-05-03", "venue": { "name": "IJCAI 2023", "status": "conference" }, "authors": [ "Jiahui Geng", "Zongxiong Chen", "Yuandou Wang", "Herbert Woisetschlaeger", "Sonja Schimmler", "Ruben Mayer", "Zhiming Zhao", "Chunming Rong" ], "kind": "survey", "settings": [ "image-classification" ], "families": [], "tier": "core", "links": { "code": "https://github.com/Guang000/Awesome-Dataset-Distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "This survey proposes a taxonomy of dataset distillation approaches and systematically reviews the methods, data modalities, and applications (continual learning, neural architecture search, privacy protection) covered by the field to date, and summarizes open challenges and future directions.", "abstract": "Dataset distillation is attracting more attention in machine learning as training sets continue to grow and the cost of training state-of-the-art models becomes increasingly high. By synthesizing datasets with high information density, dataset distillation offers a range of potential applications, including support for continual learning, neural architecture search, and privacy protection. Despite recent advances, we lack a holistic understanding of the approaches and applications. Our survey aims to bridge this gap by first proposing a taxonomy of dataset distillation, characterizing existing approaches, and then systematically reviewing the data modalities, and related applications. In addition, we summarize the challenges and discuss future directions for this field of research.", "notes": "" }, { "id": "glad-2023", "type": "paper", "title": "Generalizing Dataset Distillation via Deep Generative Prior", "short": "GLaD", "arxiv": "2305.01649", "url": null, "date": "2023-05-02", "venue": { "name": "CVPR 2023", "status": "conference" }, "authors": [ "George Cazenavette", "Tongzhou Wang", "Antonio Torralba", "Alexei A. Efros", "Jun-Yan Zhu" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "generative-latent-optimization" ], "facets": { "parameterization": [ "generative-latent" ], "labels": "hard", "prior": [ "gan" ], "regime": "single-level", "scale": [ "mnist-cifar", "imagenet-subsets" ] }, "tier": "landmark", "links": { "code": "https://github.com/georgecazenavette/glad", "project": "https://georgecazenavette.github.io/glad/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Distills into a small number of intermediate feature vectors in a pretrained deep generative model's latent space instead of pixels, augmenting existing matching-based distillation objectives with this generative prior; significantly improves cross-architecture generalization across settings and is among the first to combine a generative prior with dataset-distillation matching objectives at this scale.", "abstract": "Dataset Distillation aims to distill an entire dataset's knowledge into a few synthetic images. The idea is to synthesize a small number of synthetic data points that, when given to a learning algorithm as training data, result in a model approximating one trained on the original data. Despite recent progress in the field, existing dataset distillation methods fail to generalize to new architectures and scale to high-resolution datasets. To overcome the above issues, we propose to use the learned prior from pre-trained deep generative models to synthesize the distilled data. To achieve this, we present a new optimization algorithm that distills a large number of images into a few intermediate feature vectors in the generative model's latent space. Our method augments existing techniques, significantly improving cross-architecture generalization in all settings.", "explained": { "before": "Pixel-space matching methods DC (gradient matching), DM (distribution matching) and MTT (trajectory matching) all parameterize the synthetic set as raw pixels; they overfit to the backbone architecture used during distillation and degrade into high-frequency noise above roughly 128x128, so cross-architecture generalization is their biggest weakness. IT-GAN (concurrent) had shown latent optimization can beat pixel optimization on CIFAR with a target-trained BigGAN, but only for one matching objective.", "problem": "Directly optimizing pixels lets the synthetic set fit high-frequency, architecture-specific patterns of the one backbone network used during distillation, which do not transfer to other architectures, and this failure gets worse as resolution grows.", "idea": "Regularize any existing matching-based distillation objective by parameterizing the synthetic images as the output of an intermediate layer of a frozen, pretrained deep generative model (GLaD) rather than raw pixels; an intermediate layer balances realism (too early = restrictive) against expressivity (too late = no regularization), and, surprisingly, the generator does not need to be trained on the target dataset at all for the regularization to help.", "method": "A synthetic sample is $G(z)$ for a frozen StyleGAN-XL generator $G$ and a learnable latent $z$ that concatenates an intermediate \"F_n\" feature map (the n-th layer of the synthesis network) with the W+ modulation codes of all subsequent layers; $z$ is optimized by backpropagating whichever matching loss is being used ($\\mathcal{L}_{DC}$, $\\mathcal{L}_{DM}$, or $\\mathcal{L}_{MTT}$, computed exactly as in the underlying method) through $G$, which is otherwise treated as a plug-in add-on (Algorithm 1). $z$ is initialized either by a partial feed-forward pass through $G$ using the empirical latent distribution of the target class (when $G$ is trained on/near the target data) or by Gaussian noise matched in mean/variance (needed to make out-of-distribution or randomly-initialized generators usable). A gradient-checkpointing trick recomputes the generator's forward pass a second time only when computing $\\partial\\mathcal{S}/\\partial\\mathcal{Z}$, trading one extra forward pass for large memory savings (about 22GB per synthetic image at 128x128 in F0 space). The generator itself is never fine-tuned.", "evidence": "ImageNet subsets at 128x128, IPC=1, cross-architecture average over AlexNet/ResNet-18/VGG-11/ViT-B/16 trained on sets distilled with a Depth-5 ConvNet backbone (Table 1): GLaD improves every one of MTT/DC/DM on every subset, e.g. ImageNet-A: MTT 33.4%->39.9%, DC 38.7%->41.8%, DM 27.2%->31.6%. At IPC=10 (Table 2) gains persist but shrink, e.g. DC ImageNet-B 45.1%->50.1%, DM ImageNet-C 47.5%->49.7%. On CIFAR-10 (Table 3, IPC=1) GLaD with a StyleGAN-XL trained on CIFAR-10 raises the 4-architecture average for MTT from 24.1% to 28.0% and for DM from 22.6% to 23.8%, but only marginally for DC (26.0%->26.3%). Labels are hard throughout (all three base methods are unmodified apart from the parameterization); training recipe is SGD with momentum, 500 epochs linear warmup + 500 epochs cosine decay, 5 seeds. Key ablation isolating the mechanism (Figure 3, Table 4): (a) sweeping which layer $F_n$ is used shows an inverted-U in cross-architecture accuracy, so an intermediate space, not the earliest or latest, is what helps; (b) at 256x256 with DC, generators trained on ImageNet, on unrelated FFHQ faces, on Pokémon, or left at random initialization all give comparable cross-architecture improvements over pixel space (e.g. ImageNet-B: pixel 32.8% vs ImageNet-G 41.5% vs Random-G 40.3% vs Pokemon-G 39.4% vs FFHQ-G 40.2%), showing the gain comes mostly from the generator's architectural/manifold prior, not from it having seen the target distribution.", "limitations": "A second forward+backward pass through StyleGAN-XL is required per iteration even with checkpointing, which the authors state doubles the time overhead of an already slow generator pass, and large synthetic sets need multiple sub-batches, adding further overhead; only IPC=1/10 are tested, not larger budgets; only hard labels are used (the paper explicitly declines to use TESLA's soft-label assignment to isolate the generative-prior effect); the largest dataset tested is 10-class ImageNet subsets up to 512x512, not full ImageNet-1K; gains on CIFAR-10/DC are only marginal, so the benefit is resolution- and method-dependent, not universal." }, "builds_on": [ "dc-2021", "dm-2023", "mtt-2022", "tesla-2023", "it-gan-2022" ], "awesome_sections": [ "Main / Generative Distillation / GAN" ] }, { "id": "lcmat-2023", "type": "paper", "title": "Loss-Curvature Matching for Dataset Selection and Condensation", "short": "LCMat", "arxiv": "2303.04449", "url": null, "date": "2023-03-08", "venue": { "name": "AISTATS 2023", "status": "conference" }, "authors": [ "Seungjae Shin", "Heesun Bae", "Donghyeok Shin", "Weonyoung Joo", "Il-Chul Moon" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "gradient-matching", "coreset-selection" ], "tier": "catalogued", "links": { "code": "https://github.com/SJShin-AI/LCMat", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Matches the loss curvature — not just the loss value or a first-order gradient — of the original and reduced datasets over a neighborhood of the model-parameter space, deriving an implementable upper bound on the worst-case curvature gap, and applies the same objective to both coreset selection and dataset condensation, improving generalization over existing baselines in both settings.", "abstract": "Training neural networks on a large dataset requires substantial computational costs. Dataset reduction selects or synthesizes data instances based on the large dataset, while minimizing the degradation in generalization performance from the full dataset. Existing methods utilize the neural network during the dataset reduction procedure, so the model parameter becomes important factor in preserving the performance after reduction. By depending upon the importance of parameters, this paper introduces a new reduction objective, coined LCMat, which Matches the Loss Curvatures of the original dataset and reduced dataset over the model parameter space, more than the parameter point. This new objective induces a better adaptation of the reduced dataset on the perturbed parameter region than the exact point matching. Particularly, we identify the worst case of the loss curvature gap from the local parameter region, and we derive the implementable upper bound of such worst-case with theoretical analyses. Our experiments on both coreset selection and condensation benchmarks illustrate that LCMat shows better generalization performances than existing baselines.", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ], "notes": "Family corrected from the draft's trajectory-matching guess to gradient-matching (curvature is a second-order generalization of a single-point gradient match, not a multi-step trajectory match); also touches coreset-selection since the method targets both paradigms — flagged for review." }, { "id": "dim-2023", "type": "paper", "title": "DiM: Distilling Dataset into Generative Model", "short": "DiM", "arxiv": "2303.04707", "url": null, "date": "2023-03-08", "venue": { "name": "arXiv 2023", "status": "preprint" }, "authors": [ "Kai Wang", "Jianyang Gu", "Daquan Zhou", "Zheng Zhu", "Wei Jiang", "Yang You" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "generator-as-dataset" ], "facets": { "parameterization": [ "generator-weights" ], "labels": "hard", "prior": [ "gan" ], "regime": "generator-fine-tuning", "scale": [ "mnist-cifar" ] }, "tier": "core", "links": { "code": "https://github.com/vimar-gu/DiM", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Trains a generative model to store the target dataset by minimizing the difference in logits, from a pool of models, between real and generated images, so the same trained generator produces samples for any distillation ratio and large architectures without re-optimizing; reports state-of-the-art results on four datasets, including higher accuracy on complex architectures (75.1% ResNet-18) than simple ones (72.6% ConvNet-3) at 10 images per class on CIFAR-10.", "abstract": "Dataset distillation reduces the network training cost by synthesizing small and informative datasets from large-scale ones. Despite the success of the recent dataset distillation algorithms, three drawbacks still limit their wider application: i). the synthetic images perform poorly on large architectures; ii). they need to be re-optimized when the distillation ratio changes; iii). the limited diversity restricts the performance when the distillation ratio is large. In this paper, we propose a novel distillation scheme to \\textbf{D}istill information of large train sets \\textbf{i}nto generative \\textbf{M}odels, named DiM. Specifically, DiM learns to use a generative model to store the information of the target dataset. During the distillation phase, we minimize the differences in logits predicted by a models pool between real and generated images. At the deployment stage, the generative model synthesizes various training samples from random noises on the fly. Due to the simple yet effective designs, the trained DiM can be directly applied to different distillation ratios and large architectures without extra cost. We validate the proposed DiM across 4 datasets and achieve state-of-the-art results on all of them. To the best of our knowledge, we are the first to achieve higher accuracy on complex architectures than simple ones, such as 75.1\\% with ResNet-18 and 72.6\\% with ConvNet-3 on ten images per class of CIFAR-10. Besides, DiM outperforms previous methods with 10\\% $\\sim$ 22\\% when images per class are 1 and 10 on the SVHN dataset.", "awesome_sections": [ "Main / Generative Distillation / GAN" ], "notes": "Reclassified from generative-latent-optimization to generator-as-dataset per docs/taxonomy.md, which explicitly names DiM as a generator-as-dataset example.", "explained": { "before": "Bilevel/surrogate DiI (\"distill into images\") methods — DC, DSA, DM, CAFE, MTT, FRePo, IDC, HaBa — all store the dataset as a fixed set of images, distilled with a matching objective computed through a 3-layer ConvNet. IT-GAN had already shown that optimizing a frozen GAN's latents with a matching loss beats pixel optimization, but its stored artifact is still one latent per image, so it must be re-optimized whenever the image budget (IPC) changes.", "problem": "DiI methods have two coupled failures the paper attributes to storing images rather than a generative process: synthetic images distilled with a small ConvNet transfer poorly to larger unseen architectures (ResNet, VGG, DenseNet), and any change to the IPC/distillation-ratio budget requires re-running the whole distillation (8.3 GPU-hours for MTT, 18.2 for IDC to go from IPC=1 to 10 on CIFAR-10).", "idea": "Store the dataset in the weights of a conditional generator instead of in images: train the generator so that, for a batch of real and generator-sampled images with matched labels, a randomly chosen classifier from a pool gives it the same logits as it gives the real images. Because the artifact is a generator, any number of samples at any budget can be drawn from it after one training run.", "method": "A conditional GAN $\\mathcal{G}(\\cdot;\\beta)$ is first trained for $N{=}120$ epochs with the vanilla GAN loss $L_g$ alone (Adam, lr 1e-4) so it produces valid images; then a Logits Matching loss $L_m=\\mathrm{MSE}(m(\\mathcal{S}_{\\text{DiM}}), m(I_r))$ is added, where $m$ is a model freshly sampled each epoch from a \"models pool\" of randomly-initialized architectures (ConvNet-3, ResNet-10, ResNet-18) trained on the target dataset, and $\\mathcal{S}_{\\text{DiM}}=\\mathcal{G}([\\mathcal{Z}\\oplus\\mathcal{Y}];\\beta)$ is a batch of generated images conditioned on random noise $\\mathcal{Z}$ and one-hot labels $\\mathcal{Y}$. The generator is updated by $\\beta \\leftarrow \\arg\\min_\\beta (L_g+\\lambda L_m)$ with $\\lambda{=}0.01$; nothing besides the models pool and the frozen GAN's own training data is used — no external pretrained teacher, no diffusion or LDM. Because logits are class-wise soft targets from real images, they double as soft supervision, but a fresh model is trained from these generated images at evaluation with hard one-hot labels (the paper never keeps or reuses the pool's logits as teacher labels at deployment). At deployment, images are sampled from the trained generator on the fly for any chosen Image-Number-Per-Class (INPC), the paper's budget unit since IPC undercounts a generator's capacity.", "evidence": "MNIST/FashionMNIST/SVHN/CIFAR-10, INPC=1/10/50, ConvNet-3-trained pool, hard labels, Table 2: DiM beats Conditional GAN and every listed DiI/coreset baseline at every INPC, e.g. CIFAR-10 INPC=1 DiM 51.3% vs. MTT 46.3% vs. FRePo 46.8% vs. Conditional GAN 46.4% (real-data upper bound 84.8%); CIFAR-10 INPC=10 DiM 66.2% vs. FRePo 65.5%; SVHN INPC=1 DiM 80.9% vs. best DiI baseline ~58.5% (MTT), a ~10-22 point gain the abstract highlights. Cross-architecture (Table 7, CIFAR-10 INPC=10, distilled with ConvNet-3): DiM is the first method where ResNet-18 (69.2%) beats ConvNet-3 (66.2%) on its own synthetic data, vs. MTT dropping from 64.3% (ConvNet-3) to 46.4% (ResNet-18). Redeployment cost (Fig. 1d): re-deploying IPC 1->10 costs DiM ~0.1 GPU-hours vs. MTT 8.3 and IDC 18.2 (13x-160x cheaper), because DiM need only sample more images, not re-optimize. Key ablation (Table 4): replacing logits matching with feature matching or gradient matching inside the same DiM pipeline drops CIFAR-10 INPC=50 ResNet-18 accuracy from 75.0% to 73.8% (feature) or 71.7% (gradient), isolating logits matching (not the generator alone) as the mechanism; Table 6 further shows accuracy rises monotonically with the diversity of the models pool (single arch 73.1-74.2% avg vs. all three archs 75.0%/73.9% avg).", "limitations": "Tested only at MNIST/FashionMNIST/SVHN/CIFAR-10 resolution; the conditional GAN is trained from scratch on the exact target dataset for each experiment, so (like IT-GAN) none of the reported gain isolates a generator not trained on the target distribution; evaluation is hard-label only, with the models-pool logits used purely as a matching signal during synthesis, not carried forward as soft teacher labels at deployment; the paper's own stated limitation is that DiM must generate images on the fly at deployment, adding a small but nonzero per-batch cost (Table 9: 2ms generation vs. 8-80ms training per batch, 2%-20% overhead); it does not test ImageNet-scale data (explicitly named as future work), and the \"13x-160x\" redeployment saving excludes the upfront GAN-pretraining cost (120 GAN epochs) from the comparison." } }, { "id": "federated-learning-on-virtual-2024", "type": "paper", "title": "Federated Learning on Virtual Heterogeneous Data with Local-global Distillation", "short": "FedLGD", "arxiv": "2303.02278", "url": null, "date": "2023-03-04", "venue": { "name": "TMLR 2024", "status": "journal" }, "authors": [ "Chun-Yin Huang", "Ruinan Jin", "Can Zhao", "Daguang Xu", "Xiaoxiao Li" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "distribution-matching", "gradient-matching" ], "applications": [ "federated-learning" ], "tier": "catalogued", "links": { "code": "https://github.com/ubc-tea/FedLGD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Observes that distilling each client's local data independently can worsen federated heterogeneity, so FedLGD instead uses iterative distribution matching to inpaint global information into local synthetic (\"virtual\") data and federated gradient matching to distill global virtual anchor points that rectify heterogeneous local training. Reports outperforming prior heterogeneous-FL methods on benchmark and real-world datasets, including at large numbers of class-imbalanced clients.", "abstract": "While Federated Learning (FL) is gaining popularity for training machine learning models in a decentralized fashion, numerous challenges persist, such as asynchronization, computational expenses, data heterogeneity, and gradient and membership privacy attacks. Lately, dataset distillation has emerged as a promising solution for addressing the aforementioned challenges by generating a compact synthetic dataset that preserves a model's training efficacy. However, we discover that using distilled local datasets can amplify the heterogeneity issue in FL. To address this, we propose Federated Learning on Virtual Heterogeneous Data with Local-Global Dataset Distillation (FedLGD), where we seamlessly integrate dataset distillation algorithms into FL pipeline and train FL using a smaller synthetic dataset (referred as virtual data). Specifically, to harmonize the domain shifts, we propose iterative distribution matching to inpaint global information to local virtual data and use federated gradient matching to distill global virtual data that serve as anchor points to rectify heterogeneous local training, without compromising data privacy. We experiment on both benchmark and real-world datasets that contain heterogeneous data from different sources, and further scale up to an FL scenario that contains a large number of clients with heterogeneous and class-imbalanced data. Our method outperforms state-of-the-art heterogeneous FL algorithms under various settings. Our code is available at https://github.com/ubc-tea/FedLGD.", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "" }, { "id": "dream-2023", "type": "paper", "title": "DREAM: Efficient Dataset Distillation by Representative Matching", "short": "DREAM", "arxiv": "2302.14416", "url": null, "date": "2023-02-28", "venue": { "name": "ICCV 2023", "status": "conference" }, "authors": [ "Yanqing Liu", "Jianyang Gu", "Kai Wang", "Zheng Zhu", "Wei Jiang", "Yang You" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/lyq312318224/DREAM", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Replaces the random sampling of real images used as matching targets in gradient-, distribution- and trajectory-matching distillation with a representative-sample selection strategy chosen for even coverage and diversity, reducing the number of distillation iterations needed by more than 8x and improving final accuracy when trained longer.", "abstract": "Dataset distillation aims to synthesize small datasets with little information loss from original large-scale ones for reducing storage and training costs. Recent state-of-the-art methods mainly constrain the sample synthesis process by matching synthetic images and the original ones regarding gradients, embedding distributions, or training trajectories. Although there are various matching objectives, currently the strategy for selecting original images is limited to naive random sampling. We argue that random sampling overlooks the evenness of the selected sample distribution, which may result in noisy or biased matching targets. Besides, the sample diversity is also not constrained by random sampling. These factors together lead to optimization instability in the distilling process and degrade the training efficiency. Accordingly, we propose a novel matching strategy named as \\textbf{D}ataset distillation by \\textbf{RE}present\\textbf{A}tive \\textbf{M}atching (DREAM), where only representative original images are selected for matching. DREAM is able to be easily plugged into popular dataset distillation frameworks and reduce the distilling iterations by more than 8 times without performance drop. Given sufficient training time, DREAM further provides significant improvements and achieves state-of-the-art performances.", "awesome_sections": [ "Main / Better Optimization" ] }, { "id": "rcig-2023", "type": "paper", "title": "Dataset Distillation with Convexified Implicit Gradients", "short": "RCIG", "arxiv": "2302.06755", "url": null, "date": "2023-02-13", "venue": { "name": "ICML 2023", "status": "conference" }, "authors": [ "Noel Loo", "Ramin Hasani", "Mathias Lechner", "Daniela Rus" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "kernel-closed-form" ], "facets": { "parameterization": [ "pixels" ], "labels": "learned", "prior": [ "random-networks", "ntk-kernel" ], "regime": "bilevel", "scale": [ "imagenet-subsets", "imagenet-1k", "tiny-imagenet" ] }, "tier": "notable", "links": { "code": "https://github.com/yolky/RCIG", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Formulates dataset distillation as a bi-level problem solved with implicit gradients, convexifies the meta-gradient computation by learning on top of a frozen finite-width neural tangent kernel, and reduces implicit-gradient bias by analytically computing final-layer parameters from the body parameters; reports a 108% average improvement over prior distillation algorithms at one image per class on resized ImageNet, 66% on Tiny-ImageNet and 37% on CIFAR-100.", "abstract": "We propose a new dataset distillation algorithm using reparameterization and convexification of implicit gradients (RCIG), that substantially improves the state-of-the-art. To this end, we first formulate dataset distillation as a bi-level optimization problem. Then, we show how implicit gradients can be effectively used to compute meta-gradient updates. We further equip the algorithm with a convexified approximation that corresponds to learning on top of a frozen finite-width neural tangent kernel. Finally, we improve bias in implicit gradients by parameterizing the neural network to enable analytical computation of final-layer parameters given the body parameters. RCIG establishes the new state-of-the-art on a diverse series of dataset distillation tasks. Notably, with one image per class, on resized ImageNet, RCIG sees on average a 108\\% improvement over the previous state-of-the-art distillation algorithm. Similarly, we observed a 66\\% gain over SOTA on Tiny-ImageNet and 37\\% on CIFAR-100.", "awesome_sections": [ "Main / Kernel-Based Distillation" ], "builds_on": [ "rfad-2022", "frepo-2022" ], "explained": { "before": "Responds directly to frepo-2022 (whose loss it explicitly generalizes) and to kip-2021/rfad-2022's exact/approximate NTK approach, and to dd-2018-style BPTT and gradient/trajectory-matching surrogates (dc-2021, mtt-2022), framing all of them as either expensive-to-unroll or introducing their own matching bias.", "problem": "Bi-level dataset distillation needs the meta-gradient of an outer loss with respect to the inner learner's parameters; implicit gradients (via the implicit function theorem) offer a way to get this without unrolling, but the paper shows \"vanilla\" implicit gradients (VIG) perform poorly -- sometimes worse than random real images -- because deep networks are non-convex and the implicit-function theorem's unique-minimizer assumption fails, and diagnoses this as the specific reason implicit gradients had not previously worked for distillation.", "idea": "Make implicit gradients work by making the inner problem actually convex: linearize the network (first-order Taylor expansion around initialization, i.e. train on the frozen finite-width NTK/ conjugate kernel), which convexifies the inner objective; then further reduce bias by reparameterizing so the final layer's optimal weights are computed analytically from the body parameters at every step, rather than left to imperfect implicit-gradient optimization.", "method": "Splits network parameters into body $\\theta_B$ and final layer $\\theta_F$; under linearized (NTK) dynamics, $\\theta_F^*$ has a closed form given $\\theta_B$ (Eq. 3-4, computed via KRR on the network's own hidden features, offset by the body's contribution), so only $\\theta_B$ is treated as the inner variable. The meta-gradient combines a direct term ($\\partial\\mathcal{L}_o/\\partial\\psi$, non-zero because $\\psi$ affects $\\theta_F^*$ -- this direct term reduces to the FRePo loss when using MSE) and an implicit term computed by approximating $v=H_S^{-1}g_T$ via SGD on a Neumann-series-equivalent auxiliary loss (Hessian-vector products via the Pearlmutter trick). A pool of $m=30$ partially trained models (max $K=100$ steps before reinit, matching FRePo's scheme) supplies diversity in $\\theta_B$ across steps. A Platt-scaling (learnable-temperature cross-entropy) outer loss replaces MSE. For support sets exceeding 1,000 images, a bias-free subsampling trick runs the full forward pass but backpropagates through only a random subset of nodes, since gradient contributions from each support image are interchangeable. Ablation architecture: VIG (naive implicit gradients) -> CIG (+ linearization/ convexification) -> RCIG (+ reparameterized analytic final layer), isolating each component's contribution (Table 1).", "evidence": "Ablation (Table 1, CIFAR-100 1 IPC): random subset 6.2%, VIG 17.8%, CIG 19.7%, RCIG 39.6% -- showing convexification alone helps modestly but the reparameterized analytic final layer drives most of the gain. Headline (Table 2): CIFAR-100 x 1 IPC x 39.3+/-0.4% vs. prior SOTA (FRePo) 28.7% (37% relative gain, matches abstract); Tiny-ImageNet x 1 IPC x 25.6+/-0.3% vs. FRePo 15.4% (66% relative gain); resized ImageNet-1K x 1 IPC x 15.6+/-0.2% vs. FRePo 7.5% (108% relative gain, i.e. roughly doubling SOTA, matching the abstract's headline number). RCIG wins 13/16 standard-benchmark settings (Table 2) but underperforms FRePo on CUB-200 (12.1% vs. 12.4% at 1 IPC), which the paper attributes to overfitting on CUB-200's small training set (5,994 images across 200 classes) since RCIG uses no data augmentation. Cross-architecture transfer (Table 3, CIFAR-10 x 10 IPC): RCIG trained with BatchNorm reaches 54.4-62.1% on unseen ResNet/VGG/AlexNet architectures, comparable to or better than FRePo (52.4-61.6%) and clearly ahead of DSA/DM/MTT/KIP; trained with no normalization (the paper's default), RCIG transfers less well to BN/VGG architectures (46.2-51.3%) than the BN-trained variant, isolating training-time normalization choice (not the RCIG mechanism itself) as what drives cross-architecture robustness. ImageNet subsets (Table 4): ImageNette/ImageWoof 1 IPC 53.0%/33.9% vs. FRePo 48.1%/29.7%, though gains shrink at 10 IPC (roughly matching FRePo). Cost: reported on RTX A6000 GPUs (48GB) and RTX 4090; memory is higher than other methods because linearization roughly doubles memory, and for large support sets (e.g. CIFAR-100 at 50 IPC = 5,000 images) the subsampling trick is required to fit in memory (e.g. n_subset=2000 uses ~20Gb-scale memory per the appendix's Figure 5 sweep, with only a 1.4-point accuracy cost between the smallest and largest subsample sizes tested).", "limitations": "States explicitly that gains shrink as the support-set size grows (e.g. only a 14-point gap over SOTA at Tiny-ImageNet 10 IPC vs. a much larger gap at 1 IPC), hypothesized to be because the inner objective is harder to optimize at larger set sizes and would need a lower learning rate and more inner steps; this is the same \"matching methods help most at very small IPC\" pattern later diagnosed more generally by datm-2024. Requires full-batch (non-stochastic) gradients through the entire support set for both the inner objective and the Hessian-vector product, and linearization roughly doubles memory, so very large support sets are memory-constrained even with the subsampling workaround (which still needs a full forward pass on the whole set). Overfits on small, many-class datasets (CUB-200) because it uses no data augmentation during distillation, unlike dsa-2021/mtt-2022. Its cross-architecture robustness depends on using BatchNorm at training time, not on the RCIG mechanism per se. No result beyond resized 64x64 ImageNet-1K at 1-2 IPC, and no soft-label/teacher-relabeling scheme." } }, { "id": "understanding-reconstruction-attacks-with-2024", "type": "paper", "title": "Understanding Reconstruction Attacks with the Neural Tangent Kernel and Dataset Distillation", "arxiv": "2302.01428", "url": null, "date": "2023-02-02", "venue": { "name": "ICLR 2024", "status": "conference" }, "authors": [ "Noel Loo", "Ramin Hasani", "Mathias Lechner", "Alexander Amini", "Daniela Rus" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "kernel-closed-form" ], "applications": [ "privacy" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Strengthens the dataset-reconstruction attack on trained network parameters and proves it can recover the entire training set in the infinite-width Neural Tangent Kernel regime, showing empirically that success depends on deviation from the frozen infinite-width limit and that easily-reconstructed images tend to be dataset outliers. Turns this into a dataset distillation method by retraining on the reconstructed images, reporting high predictive accuracy from the reconstructed-then-distilled set.", "abstract": "Modern deep learning requires large volumes of data, which could contain sensitive or private information that cannot be leaked. Recent work has shown for homogeneous neural networks a large portion of this training data could be reconstructed with only access to the trained network parameters. While the attack was shown to work empirically, there exists little formal understanding of its effective regime which datapoints are susceptible to reconstruction. In this work, we first build a stronger version of the dataset reconstruction attack and show how it can provably recover the \\emph{entire training set} in the infinite width regime. We then empirically study the characteristics of this attack on two-layer networks and reveal that its success heavily depends on deviations from the frozen infinite-width Neural Tangent Kernel limit. Next, we study the nature of easily-reconstructed images. We show that both theoretically and empirically, reconstructed images tend to \"outliers\" in the dataset, and that these reconstruction attacks can be used for \\textit{dataset distillation}, that is, we can retrain on reconstructed images and obtain high predictive accuracy.", "awesome_sections": [ "Applications / Privacy" ], "notes": "" }, { "id": "fedlap-dp-2023", "type": "paper", "title": "FedLAP-DP: Federated Learning by Sharing Differentially Private Loss Approximations", "short": "FedLAP-DP", "arxiv": "2302.01068", "url": null, "date": "2023-02-02", "venue": { "name": "arXiv 2023", "status": "preprint" }, "authors": [ "Hui-Po Wang", "Dingfan Chen", "Raouf Kerkouche", "Mario Fritz" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "gradient-matching" ], "applications": [ "federated-learning", "privacy" ], "tier": "catalogued", "links": { "code": "https://github.com/a514514772/fedlap-dp", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Clients synthesize a small set of samples that approximate their local loss landscape by simulating the gradients of real images within a local region; these loss surrogates, rather than model updates, are aggregated on the server to recover the global loss landscape, which the paper shows lets record-level differential privacy be enforced at the same privacy cost as gradient-sharing FL but with a better privacy-utility trade-off and faster convergence.", "abstract": "Conventional gradient-sharing approaches for federated learning (FL), such as FedAvg, rely on aggregation of local models and often face performance degradation under differential privacy (DP) mechanisms or data heterogeneity, which can be attributed to the inconsistency between the local and global objectives. To address this issue, we propose FedLAP-DP, a novel privacy-preserving approach for FL. Our formulation involves clients synthesizing a small set of samples that approximate local loss landscapes by simulating the gradients of real images within a local region. Acting as loss surrogates, these synthetic samples are aggregated on the server side to uncover the global loss landscape and enable global optimization. Building upon these insights, we offer a new perspective to enforce record-level differential privacy in FL. A formal privacy analysis demonstrates that FedLAP-DP incurs the same privacy costs as typical gradient-sharing schemes while achieving an improved trade-off between privacy and utility. Extensive experiments validate the superiority of our approach across various datasets with highly skewed distributions in both DP and non-DP settings. Beyond the promising performance, our approach presents a faster convergence speed compared to typical gradient-sharing methods and opens up the possibility of trading communication costs for better performance by sending a larger set of synthetic images. The source is available at \\url{https://github.com/hui-po-wang/FedLAP-DP}.", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "" }, { "id": "differentially-private-kernel-inducing-2023", "type": "paper", "title": "Differentially Private Kernel Inducing Points using features from ScatterNets (DP-KIP-ScatterNet) for Privacy Preserving Data Distillation", "short": "DP-KIP-ScatterNet", "arxiv": "2301.13389", "url": null, "date": "2023-01-31", "venue": { "name": "arXiv 2023", "status": "preprint" }, "authors": [ "Margarita Vinaroz", "Mi Jung Park" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "kernel-closed-form" ], "applications": [ "privacy" ], "tier": "catalogued", "links": { "code": "https://github.com/dpclip/dpclip", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Applies DP-SGD to KIP (kernel inducing points), but finds that the convolutional NTK features KIP needs for good accuracy are too expensive to compute privately at scale; DP-KIP-ScatterNet substitutes fixed wavelet ScatterNet features, which need no pretraining (avoiding an extra privacy cost) and run on a single GPU. Reports superior performance to prior differentially-private data-distillation methods across several image datasets under formal DP guarantees.", "abstract": "Data distillation aims to generate a small data set that closely mimics the performance of a given learning algorithm on the original data set. The distilled dataset is hence useful to simplify the training process thanks to its small data size. However, distilled data samples are not necessarily privacy-preserving, even if they are generally humanly indiscernible. To address this limitation, we introduce differentially private kernel inducing points (DP-KIP) for privacy-preserving data distillation. Unlike our original intention to simply apply DP-SGD to the framework of KIP, we find that KIP using infinitely-wide convolutional neural tangent kernels (conv-NTKs) performs better compared to KIP using fully-connected NTKs. However, KIP with conv-NTKs, due to its convolutional and pooling operations, introduces an unbearable computational complexity, requiring hundreds of V100 GPUs in parallel to train, which is impractical and more importantly, such computational resources are inaccessible to many. To overcome this issue, we propose an alternative that does not require pre-training (to avoid a privacy loss) and can well capture complex information on images, as those features from conv-NKTs do, while the computational cost is manageable by a single V100 GPU. To this end, we propose DP-KIP-ScatterNet, which uses the wavelet features from Scattering networks (ScatterNet) instead of those from conv-NTKs, to perform DP-KIP at a reasonable computational cost. We implement DP-KIP-ScatterNet in -- computationally efficient -- JAX and test on several popular image datasets to show its efficacy and its superior performance compared to state-of-the art methods in image data distillation with differential privacy guarantees.", "awesome_sections": [ "Applications / Privacy" ], "notes": "" }, { "id": "survey-yu-2023", "type": "paper", "title": "Dataset Distillation: A Comprehensive Review", "short": "Yu et al. review", "arxiv": "2301.07014", "url": null, "date": "2023-01-17", "venue": { "name": "TPAMI 2023", "status": "journal" }, "authors": [ "Ruonan Yu", "Songhua Liu", "Xinchao Wang" ], "kind": "survey", "settings": [ "image-classification" ], "families": [], "tier": "core", "links": { "code": "https://github.com/Guang000/Awesome-Dataset-Distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "This review formalizes dataset distillation with an overall algorithmic framework common to existing methods, proposes a systematic taxonomy of methodologies and discusses their theoretical interconnections, and presents current challenges and future directions supported by extensive experiments.", "abstract": "Recent success of deep learning is largely attributed to the sheer amount of data used for training deep neural networks.Despite the unprecedented success, the massive data, unfortunately, significantly increases the burden on storage and transmission and further gives rise to a cumbersome model training process. Besides, relying on the raw data for training \\emph{per se} yields concerns about privacy and copyright. To alleviate these shortcomings, dataset distillation~(DD), also known as dataset condensation (DC), was introduced and has recently attracted much research attention in the community. Given an original dataset, DD aims to derive a much smaller dataset containing synthetic samples, based on which the trained models yield performance comparable with those trained on the original dataset. In this paper, we give a comprehensive review and summary of recent advances in DD and its application. We first introduce the task formally and propose an overall algorithmic framework followed by all existing DD methods. Next, we provide a systematic taxonomy of current methodologies in this area, and discuss their theoretical interconnections. We also present current challenges in DD through extensive experiments and envision possible directions for future works.", "notes": "" }, { "id": "survey-lei-2023", "type": "paper", "title": "A Comprehensive Survey of Dataset Distillation", "short": "Lei & Tao survey", "arxiv": "2301.05603", "url": null, "date": "2023-01-13", "venue": { "name": "TPAMI 2023", "status": "journal" }, "authors": [ "Shiye Lei", "Dacheng Tao" ], "kind": "survey", "settings": [ "image-classification" ], "families": [], "tier": "core", "links": { "code": "https://github.com/Guang000/Awesome-Dataset-Distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "This survey taxonomizes dataset distillation methods into meta-learning and data-matching frameworks based on whether they explicitly mimic target-data performance, covers distillation algorithms, factorized dataset distillation, performance comparisons and applications, and identifies limitations around high-resolution data and complex label spaces as open challenges.", "abstract": "Deep learning technology has developed unprecedentedly in the last decade and has become the primary choice in many application domains. This progress is mainly attributed to a systematic collaboration in which rapidly growing computing resources encourage advanced algorithms to deal with massive data. However, it has gradually become challenging to handle the unlimited growth of data with limited computing power. To this end, diverse approaches are proposed to improve data processing efficiency. Dataset distillation, a dataset reduction method, addresses this problem by synthesizing a small typical dataset from substantial data and has attracted much attention from the deep learning community. Existing dataset distillation methods can be taxonomized into meta-learning and data matching frameworks according to whether they explicitly mimic the performance of target data. Although dataset distillation has shown surprising performance in compressing datasets, there are still several limitations such as distilling high-resolution data or data with complex label spaces. This paper provides a holistic understanding of dataset distillation from multiple aspects, including distillation frameworks and algorithms, factorized dataset distillation, performance comparison, and applications. Finally, we discuss challenges and promising directions to further promote future studies on dataset distillation.", "notes": "The \"TITLE MISMATCH\" flag on the draft is resolved: the cached arXiv metadata (data/cache/arxiv.json) title matches the draft title exactly, so this is simply a verification, not a rename." }, { "id": "survey-sachdeva-2023", "type": "paper", "title": "Data Distillation: A Survey", "short": "Sachdeva & McAuley survey", "arxiv": "2301.04272", "url": null, "date": "2023-01-11", "venue": { "name": "TMLR 2023", "status": "journal" }, "authors": [ "Noveen Sachdeva", "Julian McAuley" ], "kind": "survey", "settings": [ "image-classification", "graph", "other-data" ], "families": [], "tier": "core", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "This survey presents a formal framework for data distillation with a detailed taxonomy of existing approaches, and covers the method across three data modalities: images, graphs, and user-item interactions (recommender systems), identifying current challenges and future research directions for each.", "abstract": "The popularity of deep learning has led to the curation of a vast number of massive and multifarious datasets. Despite having close-to-human performance on individual tasks, training parameter-hungry models on large datasets poses multi-faceted problems such as (a) high model-training time; (b) slow research iteration; and (c) poor eco-sustainability. As an alternative, data distillation approaches aim to synthesize terse data summaries, which can serve as effective drop-in replacements of the original dataset for scenarios like model training, inference, architecture search, etc. In this survey, we present a formal framework for data distillation, along with providing a detailed taxonomy of existing approaches. Additionally, we cover data distillation approaches for different data modalities, namely images, graphs, and user-item interactions (recommender systems), while also identifying current challenges and future research directions.", "notes": "Settings expanded from the draft's image-classification-only to include graph and other-data (recommender systems), matching the abstract's explicit modality coverage." }, { "id": "bidirectional-learning-for-offline-2023", "type": "paper", "title": "Bidirectional Learning for Offline Model-based Biological Sequence Design", "short": "BIB", "arxiv": "2301.02931", "url": null, "date": "2023-01-07", "venue": { "name": "ICML 2023", "status": "conference" }, "authors": [ "Can Chen", "Yingxue Zhang", "Xue Liu", "Mark Coates" ], "kind": "application", "settings": [ "other-data" ], "families": [ "kernel-closed-form" ], "facets": { "regime": "single-level", "prior": [ "llm" ] }, "applications": [ "black-box-optimization" ], "tier": "catalogued", "links": { "code": "https://github.com/GGchen1997/BIB-ICML2023-Submission", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Extends the bidirectional (forward/backward) offline model-based optimization idea to biological sequence design by replacing the NTK proxy with a linearized pretrained language model, giving a closed-form loss that keeps the biophysical information the NTK could not represent. A bi-level auxiliary model learns how to weight the forward and backward mappings, and a derived learning-rate-adaptation module is reported to improve DNA/protein sequence design over the NTK-only predecessor.", "abstract": "Offline model-based optimization aims to maximize a black-box objective function with a static dataset of designs and their scores. In this paper, we focus on biological sequence design to maximize some sequence score. A recent approach employs bidirectional learning, combining a forward mapping for exploitation and a backward mapping for constraint, and it relies on the neural tangent kernel (NTK) of an infinitely wide network to build a proxy model. Though effective, the NTK cannot learn features because of its parametrization, and its use prevents the incorporation of powerful pre-trained Language Models (LMs) that can capture the rich biophysical information in millions of biological sequences. We adopt an alternative proxy model, adding a linear head to a pre-trained LM, and propose a linearization scheme. This yields a closed-form loss and also takes into account the biophysical information in the pre-trained LM. In addition, the forward mapping and the backward mapping play different roles and thus deserve different weights during sequence optimization. To achieve this, we train an auxiliary model and leverage its weak supervision signal via a bi-level optimization framework to effectively learn how to balance the two mappings. Further, by extending the framework, we develop the first learning rate adaptation module \\textit{Adaptive}-$\\eta$, which is compatible with all gradient-based algorithms for offline model-based optimization. Experimental results on DNA/protein sequence design tasks verify the effectiveness of our algorithm. Our code is available~\\href{https://anonymous.4open.science/r/BIB-ICLR2023-Submission/README.md}{here.}", "awesome_sections": [ "Applications / Blackbox Optimization" ], "builds_on": [ "bidirectional-learning-for-offline-2022" ], "notes": "" }, { "id": "doorping-2023", "type": "paper", "title": "Backdoor Attacks Against Dataset Distillation", "short": "Backdoor attacks on DD", "arxiv": "2301.01197", "url": null, "date": "2023-01-03", "venue": { "name": "NDSS 2023", "status": "conference" }, "authors": [ "Yugeng Liu", "Zheng Li", "Michael Backes", "Yun Shen", "Yang Zhang" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [], "applications": [ "backdoor" ], "tier": "notable", "links": { "code": "https://github.com/liuyugeng/baadd", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Performs the first backdoor attack on models trained on dataset-distilled data, injecting triggers into the synthetic data during the distillation procedure itself rather than during model training; NAIVEATTACK adds a fixed trigger at the start of distillation, while DOORPING iteratively updates the trigger throughout the whole distillation process. Reports DOORPING reaches near-1.0 attack success rate across datasets, architectures and distillation techniques, and that the resulting backdoors evade multiple existing defense mechanisms.", "abstract": "Dataset distillation has emerged as a prominent technique to improve data efficiency when training machine learning models. It encapsulates the knowledge from a large dataset into a smaller synthetic dataset. A model trained on this smaller distilled dataset can attain comparable performance to a model trained on the original training dataset. However, the existing dataset distillation techniques mainly aim at achieving the best trade-off between resource usage efficiency and model utility. The security risks stemming from them have not been explored. This study performs the first backdoor attack against the models trained on the data distilled by dataset distillation models in the image domain. Concretely, we inject triggers into the synthetic data during the distillation procedure rather than during the model training stage, where all previous attacks are performed. We propose two types of backdoor attacks, namely NAIVEATTACK and DOORPING. NAIVEATTACK simply adds triggers to the raw data at the initial distillation phase, while DOORPING iteratively updates the triggers during the entire distillation procedure. We conduct extensive evaluations on multiple datasets, architectures, and dataset distillation techniques. Empirical evaluation shows that NAIVEATTACK achieves decent attack success rate (ASR) scores in some cases, while DOORPING reaches higher ASR scores (close to 1.0) in all cases. Furthermore, we conduct a comprehensive ablation study to analyze the factors that may affect the attack performance. Finally, we evaluate multiple defense mechanisms against our backdoor attacks and show that our attacks can practically circumvent these defense mechanisms.", "awesome_sections": [ "Applications / Privacy" ], "notes": "families left empty: attacks target the distillation procedure of arbitrary DD methods, not one mechanism family." }, { "id": "fair-graph-distillation-2023", "type": "paper", "title": "Fair Graph Distillation", "arxiv": null, "url": "https://openreview.net/forum?id=xW0ayZxPWs", "date": "2023-01-01", "venue": { "name": "NeurIPS 2023", "status": "conference" }, "authors": [ "Qizhang Feng", "Zhimeng Jiang", "Ruiquan Li", "Yicheng Wang", "Na Zou", "Jiang Bian", "Xia Hu" ], "kind": "trustworthy", "settings": [ "graph" ], "families": [ "gradient-matching" ], "applications": [ "fairness" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "openreview", "depth": "abstract" }, "summary": "Shows that GNNs trained on graphs condensed by existing graph-distillation methods can exhibit more severe group-fairness problems than GNNs trained on the original graph, then proposes a fair graph-distillation approach that generates small distilled graphs that are both fair and informative.", "abstract": null, "awesome_sections": [ "Applications / Fairness" ], "notes": "" }, { "id": "dataset-distillation-for-medical-2023", "type": "paper", "title": "Dataset Distillation for Medical Dataset Sharing", "arxiv": null, "url": "https://r2hcai.github.io/AAAI-23/pages/accepted-papers.html", "date": "2023-01-01", "venue": { "name": "AAAI 2023 Workshop", "status": "workshop" }, "authors": [ "Guang Li", "Ren Togo", "Takahiro Ogawa", "Miki Haseyama" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "trajectory-matching" ], "applications": [ "medical", "privacy" ], "tier": "catalogued", "links": { "code": "https://github.com/Guang000/mtt-distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "metadata" }, "summary": "Applies trajectory-matching dataset distillation (the linked code builds on MTT) to a medical imaging dataset to produce a small, shareable synthetic set for cross-institution medical data sharing, following the same authors' prior soft-label distillation work on gastric X-ray images.", "abstract": null, "awesome_sections": [ "Applications / Medical" ], "notes": "Landing page (workshop accepted-papers listing) gives only title, authors and affiliation, no abstract; family inferred from the linked code repository (a fork of mtt-distillation)." }, { "id": "communication-efficient-federated-skin-2023", "type": "paper", "title": "Communication-Efficient Federated Skin Lesion Classification with Generalizable Dataset Distillation", "short": "GDD-FL", "arxiv": null, "url": "https://link.springer.com/chapter/10.1007/978-3-031-47401-9_2", "date": "2023-01-01", "venue": { "name": "MICCAI 2023 Workshop", "status": "workshop" }, "authors": [ "Yuchen Tian", "Jiacheng Wang", "Yueming Jin", "Liansheng Wang" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "applications": [ "federated-learning", "medical" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "abstract" }, "summary": "GDD-FL condenses thousands of skin-lesion images per client into one synthetic image per class, modeling dataset features as an uncertain Gaussian distribution so the synthetic images capture diverse semantics and resist distribution drift across clients; only these few synthesized images are transmitted once to train a global model. Reports reduced communication cost with performance superior to classical federated learning and existing dataset-distillation baselines.", "abstract": null, "awesome_sections": [ "Applications / Medical" ], "notes": "" }, { "id": "an-efficient-dataset-condensation-2023", "type": "paper", "title": "An Efficient Dataset Condensation Plugin and Its Application to Continual Learning", "arxiv": null, "url": "https://openreview.net/forum?id=Murj6wcjRw", "date": "2023-01-01", "venue": { "name": "NeurIPS 2023", "status": "conference" }, "authors": [ "Enneng Yang", "Li Shen", "Zhenyi Wang", "Tongliang Liu", "Guibing Guo" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "factorized-bases" ] }, "applications": [ "continual-learning" ], "tier": "notable", "links": { "code": "https://github.com/EnnengYang/An-Efficient-Dataset-Condensation-Plugin", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "openreview", "depth": "abstract" }, "summary": "Observes that dataset-condensation matching objectives operate in high-dimensional pixel space even though natural images are locally connected and have lower intrinsic dimension, and proposes a plugin that instead condenses images into two low-rank matrices, matching raw and synthetic data in this low-dimensional manifold for higher condensation efficiency. Demonstrated as a continual-learning replay buffer, presented at NeurIPS 2023.", "abstract": null, "awesome_sections": [ "Applications / Continual Learning" ], "notes": "" }, { "id": "ttp-2023", "type": "paper", "title": "Few-Shot Dataset Distillation via Translative Pre-Training", "short": "Translative pre-training", "arxiv": null, "url": "https://openaccess.thecvf.com/content/ICCV2023/html/Liu_Few-Shot_Dataset_Distillation_via_Translative_Pre-Training_ICCV_2023_paper.html", "date": "2023-01", "venue": { "name": "ICCV 2023", "status": "conference" }, "authors": [ "Songhua Liu", "Xinchao Wang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization", "optimization-recipes" ], "facets": { "parameterization": [ "other" ] }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "cvf", "depth": "abstract" }, "summary": "Learns a 'distillation space' via a translator network, pretrained on large datasets with image-to-image translation, that maps synthetic images optimized cheaply in an arbitrary fixed network's space into the space of the target few-shot distillation network, needing only a few adaptation steps per new target dataset; reports about 15x faster distillation with performance comparable to iterative bi-level baselines, and satisfactory generalization across datasets, budgets and class counts.", "abstract": null, "awesome_sections": [ "Main / Distilled Dataset Parametrization" ] }, { "id": "speed-2023", "type": "paper", "title": "Sparse Parameterization for Epitomic Dataset Distillation", "short": "SPEED", "arxiv": null, "url": "https://papers.nips.cc/paper_files/paper/2023/hash/9e8889198d16fb79926e71adbe38cae4-Abstract-Conference.html", "date": "2023-01", "venue": { "name": "NeurIPS 2023", "status": "conference" }, "authors": [ "Xing Wei", "Anjia Cao", "Funing Yang", "Zhiheng Ma" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "sparse-dictionary" ] }, "tier": "catalogued", "links": { "code": "https://github.com/MIV-XJTU/SPEED", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "abstract" }, "summary": "Applies dictionary learning and sparse coding to dataset distillation: Spatial-Agnostic Epitomic Tokens and Sparse Coding Matrices represent and select the most significant features, decoded by a Feature-Recurrent Network into hierarchical, high-compression synthetic images; reports state-of-the-art results on high-resolution benchmarks and downstream applications, compatible with a variety of matching objectives.", "abstract": null, "awesome_sections": [ "Main / Distilled Dataset Parametrization" ] }, { "id": "slimmable-dc-2023", "type": "paper", "title": "Slimmable Dataset Condensation", "short": "Slimmable DC", "arxiv": null, "url": "https://openaccess.thecvf.com/content/CVPR2023/html/Liu_Slimmable_Dataset_Condensation_CVPR_2023_paper.html", "date": "2023-01", "venue": { "name": "CVPR 2023", "status": "conference" }, "authors": [ "Songhua Liu", "Jingwen Ye", "Runpeng Yu", "Xinchao Wang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "other" ] }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "cvf", "depth": "abstract" }, "summary": "Introduces slimmable dataset condensation: a significance-aware parameterization whose components can be truncated to shrink an already-condensed dataset to a smaller storage budget without re-accessing the original data, addressing the inconsistency of matching networks over time and the underdetermined solution space that make naive successive compression fail; a theoretical bound shows discarding minor components is safe, and using them as initialization for further training gives fast convergence.", "abstract": null, "awesome_sections": [ "Main / Distilled Dataset Parametrization" ] }, { "id": "mgdd-2023", "type": "paper", "title": "MGDD: A Meta Generator for Fast Dataset Distillation", "short": "MGDD", "arxiv": null, "url": "https://openreview.net/forum?id=D9CMRR5Lof", "date": "2023-01", "venue": { "name": "NeurIPS 2023", "status": "conference" }, "authors": [ "Songhua Liu", "Xinchao Wang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "generator-as-dataset" ], "facets": { "parameterization": [ "generator-weights" ], "regime": "generator-fine-tuning" }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "abstract" }, "summary": "Produces synthetic images from a generator network conditioned on a dataset-distillation initialization, with synthetic labels solved in closed form via least squares in feature space, and meta-trains the generator across many datasets so that adapting to a new target needs only a few steps; reports parity with state-of-the-art distillation baselines at 22x less computation and strong generalization to synthetic-set sizes unseen during adaptation.", "abstract": null, "awesome_sections": [ "Main / Distilled Dataset Parametrization" ], "notes": "Reclassified from parameterization to generator-as-dataset per docs/taxonomy.md, which explicitly names MGDD as a generator-as-dataset example. Abstract could not be fetched from OpenReview directly (browser-verification block); content sourced from the official NeurIPS proceedings abstract page." }, { "id": "kernel-ridge-regression-based-2023", "type": "paper", "title": "Kernel Ridge Regression-Based Graph Dataset Distillation", "short": "KIDD", "arxiv": null, "url": "https://dl.acm.org/doi/10.1145/3580305.3599398", "date": "2023-01", "venue": { "name": "KDD 2023", "status": "conference" }, "authors": [ "Zhe Xu", "Yuzhong Chen", "Menghai Pan" ], "kind": "method", "settings": [ "graph" ], "families": [ "kernel-closed-form" ], "tier": "catalogued", "links": { "code": "https://github.com/pricexu/KIDD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "acm", "depth": "abstract" }, "summary": "KIDD (Kernel ridge regression-based graph Dataset Distillation) replaces the inner GNN training loop with kernel ridge regression, using a purpose-built graph kernel (LiteGNTK, related to the random-walk graph kernel) so the condensation objective has a closed-form solution. The paper reports strong empirical performance across seven real-world graph datasets, in some cases with condensed graphs outperforming the full training set using as few as 1.65% of the original training graphs.", "abstract": null, "notes": "Non-arXiv (ACM DL, paywalled); title, mechanism and results confirmed via search excerpts of the paper and its ACM listing. Author list may be incomplete (source noted \"and others\")." }, { "id": "gradient-matching-for-categorical-2023", "type": "paper", "title": "Gradient Matching for Categorical Data Distillation in CTR Prediction", "short": "CGM", "arxiv": null, "url": "https://dl.acm.org/doi/10.1145/3604915.3608769", "date": "2023-01", "venue": { "name": "RecSys 2023", "status": "conference" }, "authors": [ "Cheng Wang", "Jiacheng Sun", "Zhenhua Dong", "Ruixuan Li", "Rui Zhang" ], "kind": "method", "settings": [ "other-data" ], "families": [ "gradient-matching" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "acm", "depth": "abstract" }, "summary": "Proposes CGM (Categorical data distillation with Gradient Matching), which extends gradient-matching dataset distillation to the high-dimensional, sparse categorical features of click-through-rate prediction data, addressing the blocked gradient flow through categorical embeddings and the cost of the resulting bi-level optimization; distills a small synthetic dataset that trains CTR models from scratch toward performance close to training on the full data.", "abstract": null, "awesome_sections": [ "Applications / Recommender Systems" ], "notes": "" }, { "id": "generative-dataset-distillation-2023", "type": "paper", "title": "Generative Dataset Distillation", "arxiv": null, "url": "https://jcicvaric.com/assets/Cicvaric2023.pdf", "date": "2023-01", "venue": { "name": "University of Tübingen 2023", "status": "preprint" }, "authors": [ "Jovan Cicvarić" ], "kind": "method", "settings": [ "image-classification", "other-data" ], "families": [ "generative-latent-optimization" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "metadata" }, "summary": "Master's thesis on generative dataset distillation by optimizing latent codes of pretrained GANs (StyleGAN2 and StyleGAN-XL) instead of pixels directly, evaluated on ImageNet-1K, CIFAR-10/100 and an imitation-learning driving task; the resulting approach placed second and won a best-paper award in the Generative Track of the 2024 Dataset Distillation Challenge.", "abstract": null, "awesome_sections": [ "Main / Thesis" ], "notes": "Non-arXiv master's thesis; exact abstract text not accessible (PDF unreadable), summary based on the author's published CV/challenge record.", "applications": [ "rl" ] }, { "id": "dataset-distillation-with-attention-2023", "type": "paper", "title": "Dataset Distillation with Attention Labels for Fine-tuning BERT", "arxiv": null, "url": "https://aclanthology.org/2023.acl-short.12/", "date": "2023-01", "venue": { "name": "ACL 2023", "status": "conference" }, "authors": [ "Aru Maekawa", "Naoki Kobayashi", "Kotaro Funakoshi", "Manabu Okumura" ], "kind": "method", "settings": [ "text" ], "families": [ "meta-learning-bptt", "label-distillation" ], "tier": "catalogued", "links": { "code": "https://github.com/arumaekawa/dataset-distillation-with-attention-labels", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "aclanthology", "depth": "abstract" }, "summary": "Proposes an attention-label distillation method for fine-tuning BERT on condensed text data, transferring attention-probability supervision from a teacher alongside a small number of learned synthetic examples in a bi-level, few-gradient-step distillation setup; reports 98.5% of full-dataset performance from a single example per class and a single gradient step, including 93.2% accuracy on AG News.", "abstract": null, "awesome_sections": [ "Applications / Text" ], "notes": "Family assignment (meta-learning-bptt + label-distillation) inferred from a secondary summary of the paper, not the primary PDF; flagged as a classification doubt for verification." }, { "id": "data-efficient-neural-network-2023", "type": "paper", "title": "Data-efficient Neural Network Training with Dataset Condensation", "arxiv": null, "url": "https://era.ed.ac.uk/handle/1842/39756", "date": "2023-01", "venue": { "name": "The University of Edinburgh 2023", "status": "preprint" }, "authors": [ "Bo Zhao" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "gradient-matching", "distribution-matching" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "metadata" }, "summary": "PhD thesis presenting the author's own gradient-matching (DC), differentiable-Siamese-augmentation (DSA) and distribution-matching (DM) approaches to dataset condensation as one body of work on data-efficient neural network training.", "abstract": null, "awesome_sections": [ "Main / Thesis" ], "notes": "Thesis compiling the author's own DC/DSA/DM papers, already catalogued individually; kept for completeness of the record.", "builds_on": [ "dc-2021", "dsa-2021", "dm-2023" ] }, { "id": "acc-dd-2023", "type": "paper", "title": "Accelerating Dataset Distillation via Model Augmentation", "short": "Acc-DD", "arxiv": "2212.06152", "url": null, "date": "2022-12-12", "venue": { "name": "CVPR 2023", "status": "conference" }, "authors": [ "Lei Zhang", "Jie Zhang", "Bowen Lei", "Subhabrata Mukherjee", "Xiang Pan", "Bo Zhao", "Caiwen Ding", "Yao Li", "Dongkuan Xu" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": "https://github.com/ncsu-dk-lab/Acc-DD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Speeds up gradient-matching dataset distillation with two model-augmentation strategies — sampling early-stage (partially trained) models and perturbing model parameters — instead of the thousands of randomly initialized models gradient matching otherwise needs, reporting up to 20x faster synthesis with accuracy comparable to prior gradient-matching methods.", "abstract": "Dataset Distillation (DD), a newly emerging field, aims at generating much smaller but efficient synthetic training datasets from large ones. Existing DD methods based on gradient matching achieve leading performance; however, they are extremely computationally intensive as they require continuously optimizing a dataset among thousands of randomly initialized models. In this paper, we assume that training the synthetic data with diverse models leads to better generalization performance. Thus we propose two model augmentation techniques, i.e. using early-stage models and parameter perturbation to learn an informative synthetic set with significantly reduced training cost. Extensive experiments demonstrate that our method achieves up to 20x speedup and comparable performance on par with state-of-the-art methods.", "awesome_sections": [ "Main / Better Optimization" ], "builds_on": [ "dc-2021" ] }, { "id": "ftd-2023", "type": "paper", "title": "Minimizing the Accumulated Trajectory Error to Improve Dataset Distillation", "short": "FTD", "arxiv": "2211.11004", "url": null, "date": "2022-11-20", "venue": { "name": "CVPR 2023", "status": "conference" }, "authors": [ "Jiawei Du", "Yidi Jiang", "Vincent Y. F. Tan", "Joey Tianyi Zhou", "Haizhou Li" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "trajectory-matching" ], "facets": { "parameterization": [ "pixels" ], "labels": "hard", "prior": [ "trained-experts" ], "regime": "single-level", "scale": [ "tiny-imagenet", "imagenet-subsets" ] }, "tier": "notable", "links": { "code": "https://github.com/AngusDujw/FTD-distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Identifies the accumulated trajectory error of trajectory-matching distillation — the mismatch between the short synthetic trajectory optimized during distillation and the long trajectory used at evaluation — and regularizes expert trajectories toward flatter minima so that weights trained on the synthetic data are more robust to this error, improving accuracy by up to 4.7 points on a higher-resolution ImageNet subset.", "abstract": "Model-based deep learning has achieved astounding successes due in part to the availability of large-scale real-world data. However, processing such massive amounts of data comes at a considerable cost in terms of computations, storage, training and the search for good neural architectures. Dataset distillation has thus recently come to the fore. This paradigm involves distilling information from large real-world datasets into tiny and compact synthetic datasets such that processing the latter ideally yields similar performances as the former. State-of-the-art methods primarily rely on learning the synthetic dataset by matching the gradients obtained during training between the real and synthetic data. However, these gradient-matching methods suffer from the so-called accumulated trajectory error caused by the discrepancy between the distillation and subsequent evaluation. To mitigate the adverse impact of this accumulated trajectory error, we propose a novel approach that encourages the optimization algorithm to seek a flat trajectory. We show that the weights trained on synthetic data are robust against the accumulated errors perturbations with the regularization towards the flat trajectory. Our method, called Flat Trajectory Distillation (FTD), is shown to boost the performance of gradient-matching methods by up to 4.7% on a subset of images of the ImageNet dataset with higher resolution images. We also validate the effectiveness and generalizability of our method with datasets of different resolutions and demonstrate its applicability to neural architecture search. Code is available at https://github.com/AngusDujw/FTD-distillation.", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ], "builds_on": [ "mtt-2022" ], "explained": { "before": "Responds to mtt-2022, which matches short synthetic-data student trajectories against precomputed expert trajectories, and to dc-2021's gradient matching, both framed as part of the same family that suffers from \"accumulated trajectory error.\"", "problem": "Names and diagnoses a gap specific to trajectory matching: a student trained on the synthetic data starts exactly at an expert checkpoint $\\theta^*_{t,0}$, but during evaluation training accumulates a growing deviation $\\epsilon_t$ from the real-data trajectory at every subsequent matching step, because the synthetic data can only be optimized to match one short segment at a time. The paper shows this accumulated error, not just short-horizon matching per se, is what caps accuracy; a naive fix (robust learning -- training experts with added weight perturbations so $\\mathcal{S}$ must tolerate more starting-point noise) helps but overloads the fixed-size synthetic set with the extra information needed to be robust, degrading convergence at a fixed budget.", "idea": "Instead of making the synthetic data robust to perturbed starting weights (which costs capacity inside the small synthetic set), make the expert trajectory itself flat -- train experts with an explicit flatness regularizer so nearby points in weight space have similar loss -- so that whatever accumulated error the student picks up during evaluation lands somewhere the trajectory is insensitive to, at no extra cost to the synthetic images.", "method": "Same trajectory-matching objective and student/expert structure as mtt-2022 (student trained on synthetic images from expert start point $\\theta^*_{t,0}$, loss = normalized distance to expert target $\\theta^*_{t,m}$), but experts are now trained by minimizing $L_{\\mathcal{T}}(f_\\theta) + \\alpha S(\\theta)$ (Eq. 13), where the sharpness term $S(\\theta)=\\max_{\\epsilon\\in\\Psi}[L_{\\mathcal{T}}(f_{ \\theta+\\epsilon})-L_{\\mathcal{T}}(f_\\theta)]$ is approximated Sharpness-Aware-Minimization-style via a single ascent step $\\hat\\epsilon=\\rho\\, g_L/\\|g_L\\|_2$ (Eq. 27). An Exponential Moving Average of expert weights during buffer-phase training is used as a separate, minor implementation detail for faster distillation convergence. What is stored: per-class synthetic images (pixels), same footprint as mtt-2022; only the expert-training procedure (a synthesis-time cache) changes.", "evidence": "Table 2 (ConvNet, hard labels; MTT value in parens): CIFAR-10 IPC1/10/50 = 46.8+/-0.3 (46.2) / 66.6+/-0.3 (65.4) / 73.8+/-0.2% (71.6) vs. full-data 84.8+/-0.1%; CIFAR-100 IPC1/10/50 = 25.2+/-0.2 (24.3) / 43.4+/-0.3 (39.7) / 50.7+/-0.3% (47.7) vs. full-data 56.2+/-0.3%; Tiny-ImageNet IPC1/10 = 10.4+/-0.3 (8.8) / 24.5+/-0.2% (23.2) vs. full-data 37.6+/-0.4%. Table 3, 128x128 ImageNet subsets (IPC1/10, MTT in parens): ImageNette 52.2+/-1.0 (47.7) / 67.7+/-0.7% (63.0), ImageWoof 30.1+/-1.0 (28.6) / 38.8+/-1.4% (35.8), ImageFruit 29.1+/-0.9 (26.6) / 44.9+/-1.5% (40.3), ImageMeow 33.8+/-1.5 (30.7) / 43.3+/-0.6% (40.4) -- the largest single gain (4.7 points, ImageNette IPC10) is the paper's headlined \"more than 4%\" ImageNet-subset improvement. Ablation: FTD's own \"MTT+Robust Learning\" baseline (perturbing expert starting weights directly, without flattening the trajectory) reaches only 45.8/63.2/72.7% on CIFAR-10 IPC1/10/50 and 24.1/39.4/47.9% on CIFAR-100 -- comparable to or below plain MTT and clearly below FTD, isolating flat-trajectory regularization (not just any form of perturbation-robustness) as the source of the gain. EMA ablation (Table, CIFAR-100/Tiny-ImageNet): \"FTD (w.o. EMA)\" reaches 43.4/49.8% (CIFAR-100 IPC10/50) and 9.8/24.1% (Tiny-ImageNet IPC1/10) vs. full FTD's 43.2/50.7% and 10.0/24.5% -- EMA's contribution is small and mixed (it can even lower CIFAR-100 IPC10 slightly), confirming flatness (not EMA) is the primary mechanism. Cross-architecture (CIFAR-10 IPC50, ConvNet-trained data evaluated unseen): FTD reaches 73.8/65.7/58.4/53.8% on ConvNet/ResNet18/VGG11/AlexNet vs. MTT's 71.6/61.9/55.4/48.2%, and vs. dc-2021's 53.9/20.8/38.8/28.7%. Downstream NAS-correlation (Table, CIFAR-10-derived proxy set of 500 images): FTD's proxy ranking correlates with real-training ranking at 0.87/0.68/0.54 (Top5/10/20) vs. MTT's 0.41/0.36/-0.04, at the same 360-minute distillation time, showing flatness also improves the synthetic set's use as a training-free NAS proxy.", "limitations": "The fix addresses robustness to accumulated error but does not remove the underlying short-rollout trajectory-matching approximation itself; still requires the same expensive cached-expert-trajectory infrastructure as mtt-2022, now with an added flatness-regularized expert-training step (run on a mix of RTX3090 and Tesla V100 GPUs, no GPU-hour figure reported for the added cost). The gains are consistent but modest at CIFAR/Tiny-ImageNet scale (roughly 1-4 points) and largest specifically on the higher-resolution ImageNet subsets; no ImageNet-1K-scale or soft-label result, so it is not directly comparable to tesla-2023's ImageNet-1K numbers." } }, { "id": "dynafed-2023", "type": "paper", "title": "DYNAFED: Tackling Client Data Heterogeneity with Global Dynamics", "short": "DynaFed", "arxiv": "2211.10878", "url": null, "date": "2022-11-20", "venue": { "name": "CVPR 2023", "status": "conference" }, "authors": [ "Renjie Pi", "Weizhong Zhang", "Yueqi Xie", "Jiahui Gao", "Xiaoyu Wang", "Sunghun Kim", "Qifeng Chen" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "trajectory-matching" ], "applications": [ "federated-learning" ], "tier": "notable", "links": { "code": "https://github.com/pipilurj/dynafed", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Instead of gathering client data, DynaFed reserves a short trajectory of global-model snapshots on the server and synthesizes a pseudo dataset so that a model trained on it mimics that trajectory's dynamics, then uses the synthetic data once to correct client drift during aggregation in later rounds. Reports effectiveness across extensive non-IID federated learning benchmarks without needing an external server-side dataset.", "abstract": "The Federated Learning (FL) paradigm is known to face challenges under heterogeneous client data. Local training on non-iid distributed data results in deflected local optimum, which causes the client models drift further away from each other and degrades the aggregated global model's performance. A natural solution is to gather all client data onto the server, such that the server has a global view of the entire data distribution. Unfortunately, this reduces to regular training, which compromises clients' privacy and conflicts with the purpose of FL. In this paper, we put forth an idea to collect and leverage global knowledge on the server without hindering data privacy. We unearth such knowledge from the dynamics of the global model's trajectory. Specifically, we first reserve a short trajectory of global model snapshots on the server. Then, we synthesize a small pseudo dataset such that the model trained on it mimics the dynamics of the reserved global model trajectory. Afterward, the synthesized data is used to help aggregate the deflected clients into the global model. We name our method Dynafed, which enjoys the following advantages: 1) we do not rely on any external on-server dataset, which requires no additional cost for data collection; 2) the pseudo data can be synthesized in early communication rounds, which enables Dynafed to take effect early for boosting the convergence and stabilizing training; 3) the pseudo data only needs to be synthesized once and can be directly utilized on the server to help aggregation in subsequent rounds. Experiments across extensive benchmarks are conducted to showcase the effectiveness of Dynafed. We also provide insights and understanding of the underlying mechanism of our method.", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "" }, { "id": "towards-robust-dataset-learning-2022", "type": "paper", "title": "Towards Robust Dataset Learning", "arxiv": "2211.10752", "url": null, "date": "2022-11-19", "venue": { "name": "arXiv 2022", "status": "preprint" }, "authors": [ "Yihan Wu", "Xinda Li", "Florian Kerschbaum", "Heng Huang", "Hongyang Zhang" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "meta-learning-bptt" ], "applications": [ "adversarial-robustness" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Poses learning a dataset such that any classifier naturally trained on it is adversarially robust as a tri-level optimization problem, avoiding the cost of adversarial training on the downstream task and showing the robustness property transfers between models and data. Proves the method learns a provably robust dataset under an abstraction that separates robust from non-robust features, and reports effectiveness across different network initializations and architectures on MNIST, CIFAR-10 and Tiny-ImageNet.", "abstract": "Adversarial training has been actively studied in recent computer vision research to improve the robustness of models. However, due to the huge computational cost of generating adversarial samples, adversarial training methods are often slow. In this paper, we study the problem of learning a robust dataset such that any classifier naturally trained on the dataset is adversarially robust. Such a dataset benefits the downstream tasks as natural training is much faster than adversarial training, and demonstrates that the desired property of robustness is transferable between models and data. In this work, we propose a principled, tri-level optimization to formulate the robust dataset learning problem. We show that, under an abstraction model that characterizes robust vs. non-robust features, the proposed method provably learns a robust dataset. Extensive experiments on MNIST, CIFAR10, and TinyImageNet demostrate the effectiveness of our algorithm with different network initializations and architectures.", "awesome_sections": [ "Applications / Robustness" ], "notes": "" }, { "id": "tesla-2023", "type": "paper", "title": "Scaling Up Dataset Distillation to ImageNet-1K with Constant Memory", "short": "TESLA", "arxiv": "2211.10586", "url": null, "date": "2022-11-19", "venue": { "name": "ICML 2023", "status": "conference" }, "authors": [ "Justin Cui", "Ruochen Wang", "Si Si", "Cho-Jui Hsieh" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "trajectory-matching" ], "facets": { "parameterization": [ "pixels" ], "labels": "soft-static", "prior": [ "trained-experts" ], "regime": "single-level", "scale": [ "imagenet-1k" ] }, "tier": "core", "links": { "code": "https://github.com/justincui03/tesla", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Derives a way to compute MTT's unrolled trajectory-matching gradient exactly with constant memory (about 6x less), letting trajectory matching scale to ImageNet-1K, and introduces a soft-label assignment that improves convergence on datasets with many classes; scales to 50 images per class on ImageNet-1K on a single GPU (versus 2 IPC for prior methods) with only a 5.9-point accuracy drop from full-data training using 4.2% of the data.", "abstract": "Dataset Distillation is a newly emerging area that aims to distill large datasets into much smaller and highly informative synthetic ones to accelerate training and reduce storage. Among various dataset distillation methods, trajectory-matching-based methods (MTT) have achieved SOTA performance in many tasks, e.g., on CIFAR-10/100. However, due to exorbitant memory consumption when unrolling optimization through SGD steps, MTT fails to scale to large-scale datasets such as ImageNet-1K. Can we scale this SOTA method to ImageNet-1K and does its effectiveness on CIFAR transfer to ImageNet-1K? To answer these questions, we first propose a procedure to exactly compute the unrolled gradient with constant memory complexity, which allows us to scale MTT to ImageNet-1K seamlessly with ~6x reduction in memory footprint. We further discover that it is challenging for MTT to handle datasets with a large number of classes, and propose a novel soft label assignment that drastically improves its convergence. The resulting algorithm sets new SOTA on ImageNet-1K: we can scale up to 50 IPCs (Image Per Class) on ImageNet-1K on a single GPU (all previous methods can only scale to 2 IPCs on ImageNet-1K), leading to the best accuracy (only 5.9% accuracy drop against full dataset training) while utilizing only 4.2% of the number of data points - an 18.2% absolute gain over prior SOTA. Our code is available at https://github.com/justincui03/tesla", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ], "builds_on": [ "mtt-2022" ], "explained": { "before": "Responds directly to mtt-2022, whose memory use scales with the number of unrolled student steps and which the paper shows was never actually run at ImageNet-1K scale; also compares against frepo-2022 and kip-2021 (kernel-closed-form), which reach ImageNet-1K but only up to 2 IPC, and notes frepo-2022's own finding that soft labels are what let a kernel method work with many classes.", "problem": "Trajectory matching backpropagates through an unrolled sequence of student SGD steps, and naive implementations store activations for every step, so memory grows with the number of unrolled steps and the synthetic batch -- this is why no trajectory-matching method had scaled past small low-resolution subsets. Separately, and independently of memory, the paper finds MTT-style matching converges poorly on datasets with many classes (ImageNet-1K's 1,000), because one-hot hard labels give the matching loss no way to share information across classes.", "idea": "(1) Reformulate the trajectory-matching gradient algebraically so it can be computed exactly, in constant memory, by processing the unrolled steps' gradient contributions sequentially rather than keeping the whole computation graph live -- an exact reformulation, not an approximation. (2) Assign each synthetic image a soft label for free, by reusing the same expert (\"teacher\") checkpoint already sampled for that iteration's trajectory-matching target to produce the label, instead of learning labels as extra optimized parameters.", "method": "Same trajectory-matching objective as mtt-2022 (student trained T steps on synthetic images from expert start point $\\theta^*_t$, loss = normalized distance to expert target $\\theta^*_{t+M}$), but the gradient of this loss with respect to each synthetic image is derived in closed form as a sum over the T steps (Eqs. 1-6) and accumulated one step at a time, so peak memory no longer scales with T. Soft Label Assignment (SLA) is train-free and hyperparameter-free: at each iteration, after sampling the target-step teacher checkpoint $\\theta^*_{t+M}$, the synthetic images are passed through it to produce soft labels, which are then used (fixed for that iteration) in the matching loss; labels are not themselves optimized as free parameters. What is stored: per-class synthetic images (pixels) plus one soft-label vector per image (from the target-step teacher, not re-derived at every downstream training epoch); expert trajectories remain a synthesis-time-only cache.", "evidence": "Table 1 (soft labels used on ImageNet-1K, ConvNet, * = soft label): CIFAR-10 IPC1/10/50 = 48.5+/-0.8 / 66.4+/-0.8 / 72.6+/-0.7% vs. full-data 86.0+/-0.1%; CIFAR-100 IPC1/10/50 = 24.8+/-0.4 / 41.7+/-0.3 / 47.9+/-0.3% vs. full-data 56.7+/-0.2%; ImageNet-1K IPC1/2/10/50 = 7.7+/-0.2* / 10.5+/-0.3* / 17.8+/-1.3* / 27.9+/-1.2% vs. full-data 33.8+/-0.3% -- a 5.9-point gap at IPC50 while using 4.2% of ImageNet-1K's data points. Crucially, in this same table MTT has no ImageNet-1K entries at all (every IPC is \"-\"): the strongest prior ImageNet-1K result is frepo-2022's kernel-closed-form method, which only reaches IPC2 (9.7+/-0.2%); TESLA's IPC50 27.9% is the reported \"18.2% absolute gain over prior SOTA\" (27.9 - 9.7), not a gain over MTT, which was never scaled to ImageNet-1K at all. Table 3 ablation (ImageNet-1K, hard vs. soft labels, IPC1/2/10/50): hard-label MTT-with-constant-memory reaches only 0.7/1.1/4.4/18.1%, vs. TESLA's soft-label 7.7/10.5/17.8/27.9% -- soft labels alone add 7-13.4 absolute points across IPCs, isolating label richness (not the memory fix) as necessary for many-class scaling. Cross-architecture (IPC10, ConvNet-trained data evaluated unseen, Table 2): ImageNet-1K ConvNet 17.8%, ResNet18 7.7%, ViT 11.0%, all exceeding Random's 3.6/1.4/3.2%; CIFAR-10 ConvNet 66.4%, ResNet18 48.9%, ViT 34.8% (vs. MTT 65.3/46.1/34.6%). Cost (Table 4): on CIFAR-100 MTT uses 17.1+/-0.1GB vs. TESLA's 3.6+/-0.1GB (4.75x reduction) at 11.5 vs. 14.5 sec/iteration (1.25x slower); on ImageNet-1K, MTT-scale memory would be 79.9+/-0.1GB vs. TESLA's 13.9+/-0.1GB (5.75x reduction) at 45.0 vs. 46.0 sec/iteration (1.02x slower) -- the constant-memory reformulation costs almost no extra time.", "limitations": "Soft labels are a fixed per-image assignment produced once during distillation from a target-step teacher checkpoint, not a teacher-relabeling scheme recomputed every downstream training epoch (the decoupled-synthesis family's later, stronger approach); the paper itself found using a fully-trained (rather than target-step) teacher for label assignment performs much worse, so the choice of which checkpoint generates labels matters and is not obviously optimal. Introducing soft labels changes what is being compared: ImageNet-1K numbers are not directly comparable to any hard-label trajectory-matching result at the same IPC, since part of the accuracy gain is attributable to the label channel rather than the images (the paper's own hard-vs-soft ablation shows this gap is large, 7-13.4 points). Still requires the same cached-expert-trajectory up-front training cost as mtt-2022; the exact-gradient reformulation trades memory for a modest (2-25%) time increase per iteration rather than reducing wall-clock cost." } }, { "id": "private-set-generation-with-2022", "type": "paper", "title": "Private Set Generation with Discriminative Information", "arxiv": "2211.04446", "url": null, "date": "2022-11-07", "venue": { "name": "NeurIPS 2022", "status": "conference" }, "authors": [ "Dingfan Chen", "Raouf Kerkouche", "Mario Fritz" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [], "applications": [ "privacy" ], "tier": "notable", "links": { "code": "https://github.com/DingfanChen/Private-Set", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Rather than fitting a full private generative model to the data distribution, directly optimizes a small set of representative samples under differential privacy, supervised by discriminative information from the downstream task, which the paper argues is an easier and more DP-training-friendly target than full-distribution generative modeling. Reports greatly improved sample utility over prior state-of-the-art differentially private generation approaches for high-dimensional data.", "abstract": "Differentially private data generation techniques have become a promising solution to the data privacy challenge -- it enables sharing of data while complying with rigorous privacy guarantees, which is essential for scientific progress in sensitive domains. Unfortunately, restricted by the inherent complexity of modeling high-dimensional distributions, existing private generative models are struggling with the utility of synthetic samples. In contrast to existing works that aim at fitting the complete data distribution, we directly optimize for a small set of samples that are representative of the distribution under the supervision of discriminative information from downstream tasks, which is generally an easier task and more suitable for private training. Our work provides an alternative view for differentially private generation of high-dimensional data and introduces a simple yet effective method that greatly improves the sample utility of state-of-the-art approaches.", "awesome_sections": [ "Applications / Privacy" ], "notes": "families left empty: abstract does not name a specific matching mechanism for the optimized representative set." }, { "id": "haba-2022", "type": "paper", "title": "Dataset Distillation via Factorization", "short": "HaBa", "arxiv": "2210.16774", "url": null, "date": "2022-10-30", "venue": { "name": "NeurIPS 2022", "status": "conference" }, "authors": [ "Songhua Liu", "Kai Wang", "Xingyi Yang", "Jingwen Ye", "Xinchao Wang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "factorized-bases" ], "labels": "hard", "prior": [ "trained-experts" ], "regime": "single-level", "scale": [ "mnist-cifar" ] }, "tier": "core", "links": { "code": "https://github.com/Huage001/DatasetFactorization", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "builds_on": [ "mtt-2022", "dc-2021", "dm-2023", "idc-2022" ], "explained": { "before": "Conventional distillation (dc-2021, dm-2023, mtt-2022) treats every synthetic sample independently and optimizes it in the original image shape, ignoring relationships between samples within a class. Concurrently, idc-2022 improves data efficiency by storing down-sampled images and upsampling at training time -- the paper notes IDC is a special case of its own framework where the hallucinator is a parameter-free upsampler.", "problem": "Treating each synthetic sample independently wastes storage: a pilot experiment shows that adding more (correlated) MTT checkpoints as extra training data lowers downstream test loss, suggesting distilled sets could be improved by exploiting sample-wise relationships rather than only by making single samples better, but naively storing more samples increases memory.", "idea": "Factorize the synthetic dataset into a small number of shared \"bases\" and several \"hallucinator\" networks; any basis fed to any hallucinator produces a distinct image, so |bases| x |hallucinators| effective images are available from a budget that stores only their sum, and an adversarial contrastive term forces different hallucinators to produce genuinely different outputs from the same basis.", "method": "Bases are image-shaped tensors (one per stored unit); hallucinators are lightweight encoder-affine transform-decoder networks (1 conv-relu block, 3-channel bottleneck) shared across all classes. A composed training image is $\\tilde x_{ij}=H_{\\theta_j}(\\hat x_i)$ for basis i and hallucinator j. The synthetic set is trained by any existing distillation loss $\\mathcal{L}_{DD}$ (trajectory matching by default, using mtt-2022's objective) plus a cosine-similarity term $\\mathcal{L}_{cos}$ that pushes apart the feature-space embeddings of images composed from the same basis but different hallucinators; a feature extractor F is trained adversarially (min for F, max for the synthetic set) via a contrastive loss, alongside a classification loss on the composed images. Labels are hard, taken from the basis's class.", "evidence": "CIFAR-10, MTT default loss, BPC (bases/class)=9 vs. IPC=10 (roughly equal storage after accounting for 5 shared hallucinators): HaBa 69.9% vs. MTT 65.3% (Table 2). CIFAR-100, BPC=49 vs. IPC=50: HaBa 47.0% vs. MTT 46.1%. SVHN, BPC=1 vs. IPC=1: HaBa 69.8% vs. MTT 58.5%. Cross-architecture (ConvNet-trained, evaluated on ResNet/VGG/AlexNet, Table 4): adding HaBa to MTT gains up to +17.6pp (AlexNet, CIFAR-10 IPC=51/BPC=50) and to DC/DM gains 1.2-12.9pp across architectures, confirming portability across three matching objectives (DC=gradient, DM=distribution, MTT=trajectory matching). Storage accounting: a hallucinator's parameters cost about as much as 2 synthetic images; with 5 shared hallucinators amortized across all classes, the paper sets BPC = IPC - 1 (one image-equivalent per class) to keep total storage close to the IPC baseline -- an approximate, not exact, equal-storage protocol. Ablation (Table 8): removing the cosine-similarity loss, the contrastive feature-extractor loss, or the task loss each costs accuracy (e.g. CIFAR-10 BPC=1: full 55.66% vs. w/o cos. 54.56%, w/o con. 54.91%, w/o task 54.62%). GPU budget: up to 4 24GB 3090s; memory is \"slightly higher\" than the underlying baseline (MTT) due to the extra losses, no exact figure given.", "limitations": "States its own limitations: the online pairwise combination of hallucinators and bases adds time and GPU memory cost versus the baseline it wraps, and it inherits that baseline's limitations (e.g. diminishing returns from adding more images at large IPC). Storage accounting for hallucinator parameters is approximate (BPC=IPC-1 as a heuristic, not an exact byte-for-byte match), which complicates precise equal-storage comparison with pixel-only baselines. Uses trajectory matching (an expert-trajectory, bi-level-adjacent objective) as its default and strongest-performing loss, so its headline numbers inherit that objective's expert-trajectory precomputation cost, which is not separately reported here. Bases keep the same spatial shape as real images by default; only limited exploration of smaller/single-channel bases is given." }, "summary": "Factorizes a distilled dataset into a small set of shared 'bases' and per-image 'hallucination networks' that recombine them into training images, a plug-and-play representation portable to any distillation baseline, with adversarial contrastive constraints on the bases and hallucinators to keep generated images diverse and discriminative; reports improved downstream accuracy while cutting compressed parameters by up to 65%, and about 10 points higher cross-architecture accuracy than baselines.", "abstract": "In this paper, we study \\xw{dataset distillation (DD)}, from a novel perspective and introduce a \\emph{dataset factorization} approach, termed \\emph{HaBa}, which is a plug-and-play strategy portable to any existing DD baseline. Unlike conventional DD approaches that aim to produce distilled and representative samples, \\emph{HaBa} explores decomposing a dataset into two components: data \\emph{Ha}llucination networks and \\emph{Ba}ses, where the latter is fed into the former to reconstruct image samples. The flexible combinations between bases and hallucination networks, therefore, equip the distilled data with exponential informativeness gain, which largely increase the representation capability of distilled datasets. To furthermore increase the data efficiency of compression results, we further introduce a pair of adversarial contrastive constraints on the resultant hallucination networks and bases, which increase the diversity of generated images and inject more discriminant information into the factorization. Extensive comparisons and experiments demonstrate that our method can yield significant improvement on downstream classification tasks compared with previous state of the arts, while reducing the total number of compressed parameters by up to 65\\%. Moreover, distilled datasets by our approach also achieve \\textasciitilde10\\% higher accuracy than baseline methods in cross-architecture generalization. Our code is available \\href{https://github.com/Huage001/DatasetFactorization}{here}.", "awesome_sections": [ "Main / Distilled Dataset Parametrization" ] }, { "id": "rfad-2022", "type": "paper", "title": "Efficient Dataset Distillation Using Random Feature Approximation", "short": "RFAD", "arxiv": "2210.12067", "url": null, "date": "2022-10-21", "venue": { "name": "NeurIPS 2022", "status": "conference" }, "authors": [ "Noel Loo", "Ramin Hasani", "Alexander Amini", "Daniela Rus" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "kernel-closed-form" ], "tier": "notable", "links": { "code": "https://github.com/yolky/RFAD", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Replaces KIP's exact O(|S|^2) neural tangent kernel computation with a random feature approximation of the Neural Network Gaussian Process kernel that scales as O(|S|), giving at least a 100x speedup that runs on a single GPU while matching KIP's accuracy across kernel-regression and finite-width training, including on model-interpretability and privacy-preservation tasks.", "abstract": "Dataset distillation compresses large datasets into smaller synthetic coresets which retain performance with the aim of reducing the storage and computational burden of processing the entire dataset. Today's best-performing algorithm, \\textit{Kernel Inducing Points} (KIP), which makes use of the correspondence between infinite-width neural networks and kernel-ridge regression, is prohibitively slow due to the exact computation of the neural tangent kernel matrix, scaling $O(|S|^2)$, with $|S|$ being the coreset size. To improve this, we propose a novel algorithm that uses a random feature approximation (RFA) of the Neural Network Gaussian Process (NNGP) kernel, which reduces the kernel matrix computation to $O(|S|)$. Our algorithm provides at least a 100-fold speedup over KIP and can run on a single GPU. Our new method, termed an RFA Distillation (RFAD), performs competitively with KIP and other dataset condensation algorithms in accuracy over a range of large-scale datasets, both in kernel regression and finite-width network training. We demonstrate the effectiveness of our approach on tasks involving model interpretability and privacy preservation.", "awesome_sections": [ "Main / Kernel-Based Distillation" ], "builds_on": [ "kip-2021" ] }, { "id": "on-divergence-measures-for-2022", "type": "paper", "title": "On Divergence Measures for Bayesian Pseudocoresets", "arxiv": "2210.06205", "url": null, "date": "2022-10-12", "venue": { "name": "NeurIPS 2022", "status": "conference" }, "authors": [ "Balhae Kim", "Jungwon Choi", "Seanie Lee", "Yoonho Lee", "Jung-Woo Ha", "Juho Lee" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "tier": "catalogued", "links": { "code": "https://github.com/balhaekim/bpc-divergences", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Casts two representative dataset-distillation algorithms as approximations to Bayesian-pseudocoreset construction that minimize reverse KL divergence and Wasserstein distance respectively, giving a unifying view of divergence measures for pseudocoreset construction, and proposes a new pseudocoreset algorithm that instead minimizes forward KL divergence; shows empirically that pseudocoresets built this way better reflect the true posterior even in high-dimensional Bayesian inference problems.", "abstract": "A Bayesian pseudocoreset is a small synthetic dataset for which the posterior over parameters approximates that of the original dataset. While promising, the scalability of Bayesian pseudocoresets is not yet validated in realistic problems such as image classification with deep neural networks. On the other hand, dataset distillation methods similarly construct a small dataset such that the optimization using the synthetic dataset converges to a solution with performance competitive with optimization using full data. Although dataset distillation has been empirically verified in large-scale settings, the framework is restricted to point estimates, and their adaptation to Bayesian inference has not been explored. This paper casts two representative dataset distillation algorithms as approximations to methods for constructing pseudocoresets by minimizing specific divergence measures: reverse KL divergence and Wasserstein distance. Furthermore, we provide a unifying view of such divergence measures in Bayesian pseudocoreset construction. Finally, we propose a novel Bayesian pseudocoreset algorithm based on minimizing forward KL divergence. Our empirical results demonstrate that the pseudocoresets constructed from these methods reflect the true posterior even in high-dimensional Bayesian inference problems.", "awesome_sections": [ "Main / Distilled Dataset Parametrization" ], "notes": "Bayesian-pseudocoreset paper, not a mainstream image-classification DD method; classified under distribution-matching because it casts DC/DM-style methods as divergence-minimizing pseudocoreset construction. Flagged as a classification doubt for senior review." }, { "id": "parameter-pruning-dd-2023", "type": "paper", "title": "Dataset Distillation Using Parameter Pruning", "short": "Parameter pruning DD", "arxiv": "2209.14609", "url": null, "date": "2022-09-29", "venue": { "name": "IEICE Transactions on Fundamentals 2023", "status": "journal" }, "authors": [ "Guang Li", "Ren Togo", "Takahiro Ogawa", "Miki Haseyama" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "optimization-recipes" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Prunes parameters that are difficult to match during the distillation process, on the premise that this yields synthetic datasets that are more robust to which parameters are matched. Reports improved distillation performance over baselines on two benchmark datasets.", "abstract": "In this study, we propose a novel dataset distillation method based on parameter pruning. The proposed method can synthesize more robust distilled datasets and improve distillation performance by pruning difficult-to-match parameters during the distillation process. Experimental results on two benchmark datasets show the superiority of the proposed method.", "notes": "" }, { "id": "no-free-lunch-privacy-2022", "type": "paper", "title": "No Free Lunch in \"Privacy for Free: How does Dataset Condensation Help Privacy\"", "short": "No Free Lunch", "arxiv": "2209.14987", "url": null, "date": "2022-09-29", "venue": { "name": "arXiv 2022", "status": "preprint" }, "authors": [ "Nicholas Carlini", "Vitaly Feldman", "Milad Nasr" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [], "applications": [ "privacy" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "A rebuttal of Dong et al. (2022, \"Privacy for Free\"), arguing its empirical evaluation and theoretical analysis of dataset condensation's privacy benefit both have major flaws, so the original paper does not provide statistically significant evidence that DC improves privacy over a naive baseline. Points out that DP-SGD, the standard privacy-preserving training approach, simultaneously gives better accuracy and a provably lower membership-attack success rate than the condensation-based claims it disputes.", "abstract": "New methods designed to preserve data privacy require careful scrutiny. Failure to preserve privacy is hard to detect, and yet can lead to catastrophic results when a system implementing a ``privacy-preserving'' method is attacked. A recent work selected for an Outstanding Paper Award at ICML 2022 (Dong et al., 2022) claims that dataset condensation (DC) significantly improves data privacy when training machine learning models. This claim is supported by theoretical analysis of a specific dataset condensation technique and an empirical evaluation of resistance to some existing membership inference attacks. In this note we examine the claims in the work of Dong et al. (2022) and describe major flaws in the empirical evaluation of the method and its theoretical analysis. These flaws imply that their work does not provide statistically significant evidence that DC improves the privacy of training ML models over a naive baseline. Moreover, previously published results show that DP-SGD, the standard approach to privacy preserving ML, simultaneously gives better accuracy and achieves a (provably) lower membership attack success rate.", "awesome_sections": [ "Applications / Privacy" ], "builds_on": [ "privacy-for-free-2022" ], "notes": "kind: analysis-theory — this is a critique/rebuttal note, not a new trust-oriented method; directly contradicts privacy-for-free-2022's central claim (see handoff notes)." }, { "id": "meta-knowledge-condensation-for-2023", "type": "paper", "title": "Meta Knowledge Condensation for Federated Learning", "arxiv": "2209.14851", "url": null, "date": "2022-09-29", "venue": { "name": "ICLR 2023", "status": "conference" }, "authors": [ "Ping Liu", "Xin Yu", "Joey Tianyi Zhou" ], "kind": "application", "settings": [ "image-classification" ], "families": [], "applications": [ "federated-learning" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Instead of exchanging model parameters, clients extract a \"meta knowledge\" representation, weighted by a dynamic per-sample assignment mechanism that adapts each training sample's contribution over the course of training, and send only this condensed representation to the server for model updates; clients also exchange meta knowledge with each other as conditional initialization to further reduce heterogeneity. Reports raising accuracy from 74.07% to 92.95% on MNIST under a restricted 10-round communication budget.", "abstract": "Existing federated learning paradigms usually extensively exchange distributed models at a central solver to achieve a more powerful model. However, this would incur severe communication burden between a server and multiple clients especially when data distributions are heterogeneous. As a result, current federated learning methods often require a large number of communication rounds in training. Unlike existing paradigms, we introduce an alternative perspective to significantly decrease the communication cost in federate learning. In this work, we first introduce a meta knowledge representation method that extracts meta knowledge from distributed clients. The extracted meta knowledge encodes essential information that can be used to improve the current model. As the training progresses, the contributions of training samples to a federated model also vary. Thus, we introduce a dynamic weight assignment mechanism that enables samples to contribute adaptively to the current model update. Then, informative meta knowledge from all active clients is sent to the server for model update. Training a model on the combined meta knowledge without exposing original data among different clients can significantly mitigate the heterogeneity issues. Moreover, to further ameliorate data heterogeneity, we also exchange meta knowledge among clients as conditional initialization for local meta knowledge extraction. Extensive experiments demonstrate the effectiveness and efficiency of our proposed method. Remarkably, our method outperforms the state-of-the-art by a large margin (from 74.07% to 92.95%) on MNIST with a restricted communication budget (i.e. 10 rounds).", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "families left empty: the \"meta knowledge\" representation is not one of the atlas mechanism families." }, { "id": "compressed-gastric-image-generation-2022", "type": "paper", "title": "Compressed Gastric Image Generation Based on Soft-Label Dataset Distillation for Medical Data Sharing", "arxiv": "2209.14635", "url": null, "date": "2022-09-29", "venue": { "name": "CMPB 2022", "status": "journal" }, "authors": [ "Guang Li", "Ren Togo", "Takahiro Ogawa", "Miki Haseyama" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "meta-learning-bptt" ], "facets": { "labels": "soft-static" }, "applications": [ "medical" ], "tier": "catalogued", "links": { "code": "https://github.com/Guang000/dataset-distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Applies soft-label dataset distillation to gastric X-ray images so that tens of thousands of images compress into a handful of anonymized synthetic images plus a distilled model a fraction of the original size, enabling medical data sharing without exposing patient images. Reports high detection performance from the small compressed set alongside large reductions in image count and stored-model size.", "abstract": "Background and objective: Sharing of medical data is required to enable the cross-agency flow of healthcare information and construct high-accuracy computer-aided diagnosis systems. However, the large sizes of medical datasets, the massive amount of memory of saved deep convolutional neural network (DCNN) models, and patients' privacy protection are problems that can lead to inefficient medical data sharing. Therefore, this study proposes a novel soft-label dataset distillation method for medical data sharing. Methods: The proposed method distills valid information of medical image data and generates several compressed images with different data distributions for anonymous medical data sharing. Furthermore, our method can extract essential weights of DCNN models to reduce the memory required to save trained models for efficient medical data sharing. Results: The proposed method can compress tens of thousands of images into several soft-label images and reduce the size of a trained model to a few hundredths of its original size. The compressed images obtained after distillation have been visually anonymized; therefore, they do not contain the private information of the patients. Furthermore, we can realize high-detection performance with a small number of compressed images. Conclusions: The experimental results show that the proposed method can improve the efficiency and security of medical data sharing.", "awesome_sections": [ "Applications / Medical" ], "notes": "" }, { "id": "bidirectional-learning-for-offline-2022", "type": "paper", "title": "Bidirectional Learning for Offline Infinite-width Model-based Optimization", "short": "BDI", "arxiv": "2209.07507", "url": null, "date": "2022-09-15", "venue": { "name": "NeurIPS 2022", "status": "conference" }, "authors": [ "Can Chen", "Yingxue Zhang", "Jie Fu", "Xue Liu", "Mark Coates" ], "kind": "application", "settings": [ "other-data" ], "families": [ "kernel-closed-form" ], "facets": { "regime": "single-level" }, "applications": [ "black-box-optimization" ], "tier": "catalogued", "links": { "code": "https://github.com/ggchen1997/bdi", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "BDI adds a backward mapping to offline model-based optimization that distills information from the static design dataset into the high-scoring candidate designs it proposes, alongside the usual forward proxy. Using an infinite-width network and its neural tangent kernel makes the backward loss closed-form, which the paper reports mitigates the out-of-distribution failure of proxy-only gradient ascent across several design tasks.", "abstract": "In offline model-based optimization, we strive to maximize a black-box objective function by only leveraging a static dataset of designs and their scores. This problem setting arises in numerous fields including the design of materials, robots, DNA sequences, and proteins. Recent approaches train a deep neural network (DNN) on the static dataset to act as a proxy function, and then perform gradient ascent on the existing designs to obtain potentially high-scoring designs. This methodology frequently suffers from the out-of-distribution problem where the proxy function often returns poor designs. To mitigate this problem, we propose BiDirectional learning for offline Infinite-width model-based optimization (BDI). BDI consists of two mappings: the forward mapping leverages the static dataset to predict the scores of the high-scoring designs, and the backward mapping leverages the high-scoring designs to predict the scores of the static dataset. The backward mapping, neglected in previous work, can distill more information from the static dataset into the high-scoring designs, which effectively mitigates the out-of-distribution problem. For a finite-width DNN model, the loss function of the backward mapping is intractable and only has an approximate form, which leads to a significant deterioration of the design quality. We thus adopt an infinite-width DNN model, and propose to employ the corresponding neural tangent kernel to yield a closed-form loss for more accurate design updates. Experiments on various tasks verify the effectiveness of BDI. The code is available at https://github.com/GGchen1997/BDI.", "awesome_sections": [ "Applications / Blackbox Optimization" ], "notes": "" }, { "id": "federated-learning-via-decentralized-2023", "type": "paper", "title": "Federated Learning via Decentralized Dataset Distillation in Resource-Constrained Edge Environments", "short": "FedD3", "arxiv": "2208.11311", "url": null, "date": "2022-08-24", "venue": { "name": "IJCNN 2023", "status": "conference" }, "authors": [ "Rui Song", "Dai Liu", "Dave Zhenyu Chen", "Andreas Festag", "Carsten Trinitis", "Martin Schulz", "Alois Knoll" ], "kind": "application", "settings": [ "image-classification" ], "families": [], "applications": [ "federated-learning" ], "tier": "catalogued", "links": { "code": "https://github.com/rruisong/fedd3", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "FedD3 needs only one-shot communication: each client distills its local dataset independently and the server aggregates the decentralized distilled datasets directly for model training, instead of exchanging model updates iteratively. Reports large communication savings over other one-shot federated learning approaches, e.g. over 71% higher accuracy at similar communication volume, or 98% less communication volume at the same accuracy, training AlexNet on non-IID CIFAR-10 with 10 clients.", "abstract": "In federated learning, all networked clients contribute to the model training cooperatively. However, with model sizes increasing, even sharing the trained partial models often leads to severe communication bottlenecks in underlying networks, especially when communicated iteratively. In this paper, we introduce a federated learning framework FedD3 requiring only one-shot communication by integrating dataset distillation instances. Instead of sharing model updates in other federated learning approaches, FedD3 allows the connected clients to distill the local datasets independently, and then aggregates those decentralized distilled datasets (e.g. a few unrecognizable images) from networks for model training. Our experimental results show that FedD3 significantly outperforms other federated learning frameworks in terms of needed communication volumes, while it provides the additional benefit to be able to balance the trade-off between accuracy and communication cost, depending on usage scenario or target dataset. For instance, for training an AlexNet model on CIFAR-10 with 10 clients under non-independent and identically distributed (Non-IID) setting, FedD3 can either increase the accuracy by over 71% with a similar communication volume, or save 98% of communication volume, while reaching the same accuracy, compared to other one-shot federated learning approaches.", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "families left empty: abstract does not name the client-side distillation mechanism." }, { "id": "kfs-2022", "type": "paper", "title": "Dataset Condensation with Latent Space Knowledge Factorization and Sharing", "short": "KFS", "arxiv": "2208.10494", "url": null, "date": "2022-08-21", "venue": { "name": "arXiv 2022", "status": "preprint" }, "authors": [ "Hae Beom Lee", "Dong Bok Lee", "Sung Ju Hwang" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "factorized-bases" ] }, "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Assumes a generative process for the distilled data -- a set of learnable latent codes in a compact space combined with several tiny decoders -- so combining codes and decoders interchangeably multiplies the number of distinct synthetic examples at nearly the same parameter count; reports new state-of-the-art results by significant margins on SVHN, CIFAR-10, CIFAR-100 and Tiny-ImageNet.", "abstract": "In this paper, we introduce a novel approach for systematically solving dataset condensation problem in an efficient manner by exploiting the regularity in a given dataset. Instead of condensing the dataset directly in the original input space, we assume a generative process of the dataset with a set of learnable codes defined in a compact latent space followed by a set of tiny decoders which maps them differently to the original input space. By combining different codes and decoders interchangeably, we can dramatically increase the number of synthetic examples with essentially the same parameter count, because the latent space is much lower dimensional and since we can assume as many decoders as necessary to capture different styles represented in the dataset with negligible cost. Such knowledge factorization allows efficient sharing of information between synthetic examples in a systematic way, providing far better trade-off between compression ratio and quality of the generated examples. We experimentally show that our method achieves new state-of-the-art records by significant margins on various benchmark datasets such as SVHN, CIFAR10, CIFAR100, and TinyImageNet.", "awesome_sections": [ "Main / Distilled Dataset Parametrization" ] }, { "id": "can-we-achieve-robustness-2022", "type": "paper", "title": "Can we achieve robustness from data alone?", "arxiv": "2207.11727", "url": null, "date": "2022-07-24", "venue": { "name": "ICML 2022 Workshop", "status": "workshop" }, "authors": [ "Nikolaos Tsilivis", "Jingtong Su", "Julia Kempe" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [ "kernel-closed-form" ], "facets": { "regime": "bilevel", "prior": [ "ntk-kernel" ] }, "applications": [ "adversarial-robustness" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Poses adversarially robust classification as a bi-level kernel-ridge-regression problem over Neural Tangent Kernels, optimizing a dataset once so that plain gradient descent on it yields a robust model with no specialized robust training algorithm. The paper reports experiments across standard vision benchmarks and models, and revisits a prior data-optimization approach to argue that robustness from standard training on an optimized dataset is harder to obtain than previously claimed.", "abstract": "We introduce a meta-learning algorithm for adversarially robust classification. The proposed method tries to be as model agnostic as possible and optimizes a dataset prior to its deployment in a machine learning system, aiming to effectively erase its non-robust features. Once the dataset has been created, in principle no specialized algorithm (besides standard gradient descent) is needed to train a robust model. We formulate the data optimization procedure as a bi-level optimization problem on kernel regression, with a class of kernels that describe infinitely wide neural nets (Neural Tangent Kernels). We present extensive experiments on standard computer vision benchmarks using a variety of different models, demonstrating the effectiveness of our method, while also pointing out its current shortcomings. In parallel, we revisit prior work that also focused on the problem of data optimization for robust classification \\citep{Ily+19}, and show that being robust to adversarial attacks after standard (gradient descent) training on a suitable dataset is more challenging than previously thought.", "awesome_sections": [ "Applications / Robustness" ], "notes": "" }, { "id": "feddm-2023", "type": "paper", "title": "FedDM: Iterative Distribution Matching for Communication-Efficient Federated Learning", "short": "FedDM", "arxiv": "2207.09653", "url": null, "date": "2022-07-20", "venue": { "name": "CVPR 2023", "status": "conference" }, "authors": [ "Yuanhao Xiong", "Ruochen Wang", "Minhao Cheng", "Felix Yu", "Cho-Jui Hsieh" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "applications": [ "federated-learning", "privacy" ], "tier": "notable", "links": { "code": "https://github.com/anonymifish/fed-distribution-matching", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Each client builds a local surrogate loss landscape by matching the distribution of a small synthetic set to its own real data, and only this synthetic data (not model weights) is transmitted so the server assembles a global training objective from multiple clients' surrogate landscapes. Reports fewer communication rounds and better model quality than other federated learning methods, and shows the synthetic-data transmission can be made differentially private under a Gaussian mechanism at the same privacy budget.", "abstract": "Federated learning~(FL) has recently attracted increasing attention from academia and industry, with the ultimate goal of achieving collaborative training under privacy and communication constraints. Existing iterative model averaging based FL algorithms require a large number of communication rounds to obtain a well-performed model due to extremely unbalanced and non-i.i.d data partitioning among different clients. Thus, we propose FedDM to build the global training objective from multiple local surrogate functions, which enables the server to gain a more global view of the loss landscape. In detail, we construct synthetic sets of data on each client to locally match the loss landscape from original data through distribution matching. FedDM reduces communication rounds and improves model quality by transmitting more informative and smaller synthesized data compared with unwieldy model weights. We conduct extensive experiments on three image classification datasets, and results show that our method can outperform other FL counterparts in terms of efficiency and model performance. Moreover, we demonstrate that FedDM can be adapted to preserve differential privacy with Gaussian mechanism and train a better model under the same privacy budget.", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "" }, { "id": "dc-bench-2022", "type": "paper", "title": "DC-BENCH: Dataset Condensation Benchmark", "short": "DC-BENCH", "arxiv": "2207.09639", "url": null, "date": "2022-07-20", "venue": { "name": "NeurIPS 2022", "status": "conference" }, "authors": [ "Justin Cui", "Ruochen Wang", "Si Si", "Cho-Jui Hsieh" ], "kind": "evaluation-benchmark", "settings": [ "image-classification" ], "families": [], "facets": { "scale": [ "mnist-cifar", "tiny-imagenet" ] }, "tier": "core", "links": { "code": "https://github.com/justincui03/dc_benchmark", "project": "https://dc-bench.github.io/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "By 2022, DC (gradient matching), DSA (differentiable Siamese augmentation), DM (distribution matching) and MTT/TM (trajectory matching) each reported their own numbers against their own choice of baselines, augmentation and architecture, and each claimed to beat coreset selection (random, K-Center/herding) by a wide margin. No shared protocol existed to check whether these comparisons were apples-to-apples.", "problem": "Condensation methods differ not only in synthesis objective but in the augmentation and architecture used to *evaluate* the resulting dataset, and these evaluation-side choices were not controlled for; the paper's own diagnosis is that condensed-dataset quality is \"shadowed\" by such factors, so reported accuracy gaps may reflect the evaluation recipe rather than the synthetic data.", "idea": "Build one benchmark that fixes the evaluation protocol (five augmentation strategies, five architectures, IPC sweeps up to 1000, and a NAS task) and reruns DC, DSA, DM and TM plus random and K-Center selection through it, so every number is directly comparable; use this to test whether condensation's advantage over selection survives once augmentation, architecture and compression ratio are no longer confounds.", "method": "No new synthesis method: DC-BENCH is an evaluation library (open-sourced) with four protocols — (1) accuracy under 5 augmentation strategies (DSA, AutoAugment, RandAugment, ImagenetAug, none), reported as no-augmentation / best-augmentation / average-across-augmentations; (2) accuracy across a sweep of compression ratios (IPC 1 to 1000 on CIFAR-10); (3) cross-architecture transfer (ConvNet, MLP, ResNet-18, ResNet-152, ViT); (4) a Neural Architecture Search task using NAS-Bench-201, measuring Spearman rank correlation between architecture rankings on condensed vs. full data. All experiments standardized to 1x NVIDIA A100 40GB per run; methods reported as OOM if they exceed memory. Also tests initializing DC/DSA/DM/TM's synthetic images from K-Center-selected images instead of random/Gaussian init.", "evidence": "All numbers below use hard labels (no soft-label/relabel regime existed yet for these small-scale methods) and are directly comparable within each table since augmentation/architecture are held fixed per row. Augmentation (Table 1, CIFAR-10 IPC10): DC's accuracy rises from 44.43% (no aug) to 47.11% (avg) to 50.99% (best aug), a 6.56-point swing from augmentation choice alone; DSA rises 9.4 points, TM 10.2 points; under best augmentation, K-Center selection (52.96%) beats DC (50.99%) and is close to DSA (52.96% vs DSA's own number) at IPC10 on CIFAR-10, and at IPC50 K-Center even beats or matches DC on CIFAR-100/TinyImageNet. Compression ratio (Fig. 2, CIFAR-10): condensation's edge over random selection is \"only obvious for IPCs less than 200\"; above IPC200, \"all methods perform similarly to the random selection baseline,\" and DM's claimed advantage over DSA at IPC50 does not hold at larger IPCs. Cross-architecture transfer (Table 2, IPC10): all synthesis methods drop when transferred off their synthesis architecture, rankings are not preserved (DC beats K-Center on ConvNet but not on ResNet), and K-Center transfers better than \"several condensation methods\"; no method transfers well to ResNet-152 (a data-hungry large model: 45.59% from 300 random images/class vs. 70.29% at IPC1000). NAS (Table 3, NAS-Bench-201, 100 sampled architectures): Spearman rank correlation between condensed-data rankings and full-dataset rankings is *negative* for DC (-0.19), DSA (-0.37), DM (-0.37) and KIP (-0.50), and only weakly positive for K-Center (0.11) and near-zero for random (-0.06) and TM (-0.09) — none reach the original dataset's 0.7487, contradicting an earlier DC paper's reported 0.79 correlation on a smaller 720-architecture toy search space. K-Center initialization for DC/DSA/DM/TM synthesis (Fig. 4) needs \"about 30% of the computation budget\" to match random-init's end accuracy, and ends 1.3 points higher on average; DSA synthesis itself takes ~15 hours at IPC50 on CIFAR-10 (cost note, Section 4.6).", "limitations": "Stated: the benchmark's scope (architectures, datasets, downstream tasks) will need to expand as the field grows; evaluated only on image classification (CIFAR-10/100, TinyImageNet), with no text/graph/audio tasks, at the time of writing. Observed: TM/MTT could not be scaled beyond IPC50 on CIFAR-10 due to memory/runtime and is excluded from the compression-ratio sweep; the NAS evaluation uses a reduced (3-block) architecture during search because full NAS-Bench-201 networks perform poorly on condensed data, which may itself bias the correlation results; only four synthesis methods (DC, DSA, DM, TM) plus KIP (kernel) are covered, predating decoupled/relabel+KD methods (SRe2L and later) entirely, so its findings about augmentation/architecture sensitivity are not shown to generalize to that later paradigm." }, "summary": "Introduces DC-BENCH, the first large-scale standardized benchmark for dataset condensation, evaluating methods under controlled data-augmentation and architecture choices so that gains attributable to the condensation method itself can be separated from evaluation-pipeline confounds, and releases evaluators, baselines and generated datasets for reproducible comparison.", "abstract": "Dataset Condensation is a newly emerging technique aiming at learning a tiny dataset that captures the rich information encoded in the original dataset. As the size of datasets contemporary machine learning models rely on becomes increasingly large, condensation methods become a prominent direction for accelerating network training and reducing data storage. Despite numerous methods have been proposed in this rapidly growing field, evaluating and comparing different condensation methods is non-trivial and still remains an open issue. The quality of condensed dataset are often shadowed by many critical contributing factors to the end performance, such as data augmentation and model architectures. The lack of a systematic way to evaluate and compare condensation methods not only hinders our understanding of existing techniques, but also discourages practical usage of the synthesized datasets. This work provides the first large-scale standardized benchmark on Dataset Condensation. It consists of a suite of evaluations to comprehensively reflect the generability and effectiveness of condensation methods through the lens of their generated dataset. Leveraging this benchmark, we conduct a large-scale study of current condensation methods, and report many insightful findings that open up new possibilities for future development. The benchmark library, including evaluators, baseline methods, and generated datasets, is open-sourced to facilitate future research and application.", "awesome_sections": [ "Main / Benchmark" ], "notes": "" }, { "id": "on-implicit-bias-in-2022", "type": "paper", "title": "On Implicit Bias in Overparameterized Bilevel Optimization", "arxiv": null, "url": "https://proceedings.mlr.press/v162/vicol22a.html", "date": "2022-07", "venue": { "name": "ICML 2022", "status": "conference" }, "authors": [ "Paul Vicol", "Jonathan P. Lorraine", "Fabian Pedregosa", "David Duvenaud", "Roger B. Grosse" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [ "meta-learning-bptt" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "pmlr", "depth": "abstract" }, "summary": "Studies how the algorithmic choices of bi-level optimization — cold-start versus warm-start inner-loop initialization, and the hypergradient approximation used — implicitly bias which solution gradient-based bi-level methods converge to, in settings including hyperparameter optimization, meta-learning and dataset distillation, and shows warm-start solutions can retain substantial information about the outer objective even in low-dimensional problems.", "abstract": null, "awesome_sections": [ "Main / Better Understanding" ] }, { "id": "graph-condensation-via-receptive-2022", "type": "paper", "title": "Graph Condensation via Receptive Field Distribution Matching", "short": "GCDM", "arxiv": "2206.13697", "url": null, "date": "2022-06-28", "venue": { "name": "arXiv 2022", "status": "preprint" }, "authors": [ "Mengyang Liu", "Shanchuan Li", "Xinshi Chen", "Le Song" ], "kind": "method", "settings": [ "graph" ], "families": [ "distribution-matching" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Proposes GCDM, which views a graph as a distribution of receptive fields and condenses it by matching that distribution between the real and synthetic graphs with a maximum-mean-discrepancy loss; reports strong cross-architecture generalization to a variety of GNN models and substantially faster condensation than earlier graph condensation methods.", "abstract": "Graph neural networks (GNNs) enable the analysis of graphs using deep learning, with promising results in capturing structured information in graphs. This paper focuses on creating a small graph to represent the original graph, so that GNNs trained on the size-reduced graph can make accurate predictions. We view the original graph as a distribution of receptive fields and aim to synthesize a small graph whose receptive fields share a similar distribution. Thus, we propose Graph Condesation via Receptive Field Distribution Matching (GCDM), which is accomplished by optimizing the synthetic graph through the use of a distribution matching loss quantified by maximum mean discrepancy (MMD). Additionally, we demonstrate that the synthetic graph generated by GCDM is highly generalizable to a variety of models in evaluation phase and that the condensing speed is significantly improved using this framework.", "awesome_sections": [ "Applications / Graph Neural Network" ], "notes": "" }, { "id": "sample-condensation-in-online-2022", "type": "paper", "title": "Sample Condensation in Online Continual Learning", "short": "OLCGM", "arxiv": "2206.11849", "url": null, "date": "2022-06-23", "venue": { "name": "IJCNN 2022", "status": "conference" }, "authors": [ "Mattia Sangermano", "Antonio Carta", "Andrea Cossu", "Davide Bacciu" ], "kind": "application", "settings": [ "image-classification" ], "families": [], "applications": [ "continual-learning" ], "tier": "catalogued", "links": { "code": "https://github.com/MattiaSangermano/OLCGM", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "OLCGM is a replay-based online continual learning strategy that continuously compresses its memory buffer with knowledge-condensation techniques as new data streams in, rather than dropping old samples outright when the buffer fills. Reports improved final accuracy over state-of-the-art replay strategies when the memory budget is small relative to the data's complexity.", "abstract": "Online Continual learning is a challenging learning scenario where the model must learn from a non-stationary stream of data where each sample is seen only once. The main challenge is to incrementally learn while avoiding catastrophic forgetting, namely the problem of forgetting previously acquired knowledge while learning from new data. A popular solution in these scenario is to use a small memory to retain old data and rehearse them over time. Unfortunately, due to the limited memory size, the quality of the memory will deteriorate over time. In this paper we propose OLCGM, a novel replay-based continual learning strategy that uses knowledge condensation techniques to continuously compress the memory and achieve a better use of its limited size. The sample condensation step compresses old samples, instead of removing them like other replay strategies. As a result, the experiments show that, whenever the memory budget is limited compared to the complexity of the data, OLCGM improves the final accuracy compared to state-of-the-art replay strategies.", "awesome_sections": [ "Applications / Continual Learning" ], "notes": "families left empty: abstract does not name the underlying condensation mechanism." }, { "id": "pranc-2023", "type": "paper", "title": "PRANC: Pseudo RAndom Networks for Compacting deep models", "short": "PRANC", "arxiv": "2206.08464", "url": null, "date": "2022-06-16", "venue": { "name": "ICCV 2023", "status": "conference" }, "authors": [ "Parsa Nooralinejad", "Ali Abbasi", "Soroush Abbasi Koohpayegani", "Kossar Pourahmadi Meibodi", "Rana Muhammad Shahroz Khan", "Soheil Kolouri", "Hamed Pirsiavash" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "other" ] }, "tier": "catalogued", "links": { "code": "https://github.com/UCDvision/PRANC", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Reparametrizes a deep model as a linear combination of many frozen, pseudo-random 'basis' networks generated from a single seed, so the model is stored as one scalar seed plus learned mixture coefficients; applied to compress image-classification models nearly 100x and, in one experiment, to compact individual images via their associated implicit neural representations, with memory-efficient on-the-fly weight generation for inference.", "abstract": "We demonstrate that a deep model can be reparametrized as a linear combination of several randomly initialized and frozen deep models in the weight space. During training, we seek local minima that reside within the subspace spanned by these random models (i.e., `basis' networks). Our framework, PRANC, enables significant compaction of a deep model. The model can be reconstructed using a single scalar `seed,' employed to generate the pseudo-random `basis' networks, together with the learned linear mixture coefficients. In practical applications, PRANC addresses the challenge of efficiently storing and communicating deep models, a common bottleneck in several scenarios, including multi-agent learning, continual learners, federated systems, and edge devices, among others. In this study, we employ PRANC to condense image classification models and compress images by compacting their associated implicit neural networks. PRANC outperforms baselines with a large margin on image classification when compressing a deep model almost $100$ times. Moreover, we show that PRANC enables memory-efficient inference by generating layer-wise weights on the fly. The source code of PRANC is here: \\url{https://github.com/UCDvision/PRANC}", "awesome_sections": [ "Main / Distilled Dataset Parametrization" ], "notes": "Primarily a model-compression paper (PRANC compacts networks in general); its image-compaction experiment is the only part relevant to dataset distillation. Flagged as a scope doubt -- borderline inclusion, kept because the Awesome list places it under dataset parameterization." }, { "id": "doscond-2022", "type": "paper", "title": "Condensing Graphs via One-Step Gradient Matching", "short": "DosCond", "arxiv": "2206.07746", "url": null, "date": "2022-06-15", "venue": { "name": "KDD 2022", "status": "conference" }, "authors": [ "Wei Jin", "Xianfeng Tang", "Haoming Jiang", "Zheng Li", "Danqing Zhang", "Jiliang Tang", "Bing Yin" ], "kind": "method", "settings": [ "graph" ], "families": [ "gradient-matching" ], "facets": { "parameterization": [ "graph-structure" ], "labels": "hard", "prior": [ "random-networks" ], "regime": "single-level" }, "tier": "catalogued", "links": { "code": "https://github.com/amazon-research/DosCond", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Proposes DosCond, a one-step gradient-matching scheme for graph condensation that models the discrete graph structure probabilistically and matches gradients for a single step instead of unrolling multi-step training; reports 90% graph-size reduction while retaining up to 98% of original performance, about 15x faster than multi-step gradient matching.", "abstract": "As training deep learning models on large dataset takes a lot of time and resources, it is desired to construct a small synthetic dataset with which we can train deep learning models sufficiently. There are recent works that have explored solutions on condensing image datasets through complex bi-level optimization. For instance, dataset condensation (DC) matches network gradients w.r.t. large-real data and small-synthetic data, where the network weights are optimized for multiple steps at each outer iteration. However, existing approaches have their inherent limitations: (1) they are not directly applicable to graphs where the data is discrete; and (2) the condensation process is computationally expensive due to the involved nested optimization. To bridge the gap, we investigate efficient dataset condensation tailored for graph datasets where we model the discrete graph structure as a probabilistic model. We further propose a one-step gradient matching scheme, which performs gradient matching for only one single step without training the network weights. Our theoretical analysis shows this strategy can generate synthetic graphs that lead to lower classification loss on real graphs. Extensive experiments on various graph datasets demonstrate the effectiveness and efficiency of the proposed method. In particular, we are able to reduce the dataset size by 90% while approximating up to 98% of the original performance and our method is significantly faster than multi-step gradient matching (e.g. 15x in CIFAR10 for synthesizing 500 graphs). Code is available at \\url{https://github.com/amazon-research/DosCond}.", "builds_on": [ "dc-2021" ], "awesome_sections": [ "Applications / Graph Neural Network" ], "notes": "", "explained": { "before": "DD (bi-level meta-learning) and DC (Zhao et al. 2021, gradient matching with multi-step unrolled inner training) work on continuous images. GCond had extended gradient matching to graphs for node classification but keeps the full multi-step bi-level inner loop and produces a weighted, real-valued adjacency matrix that must be stored in float and thresholded post hoc.", "problem": "Two obstacles block a direct port of image gradient matching to graphs: (1) the adjacency matrix is discrete (0/1), and no existing condensation method outputs binary values; (2) the multi-step bi-level inner loop used by DC/GCond has complexity that scales with the product of inner and outer iterations, which is especially costly for graphs because the GNN forward pass itself scales with the number of edges (quadratic potential in condensed node count per graph).", "idea": "Model each entry of the condensed adjacency matrix as a Bernoulli random variable (parameterized by a learnable logit, made differentiable via the binary-concrete/Gumbel-softmax reparameterization), and replace multi-step bi-level gradient matching with \"one-step\" matching that only aligns gradients at the initial random weights $\\theta_0$, discarding the training trajectory — theoretically justified by a generalization bound showing the loss gap is controlled by the initial-step gradient distance plus a norm term.", "method": "Optimizes condensed node features $\\mathbf{X}'$ and adjacency logits $\\Omega$ (with $\\mathbf{A}'_{ij}= \\sigma((\\log\\alpha-\\log(1-\\alpha)+\\Omega_{ij})/\\tau)$, $\\alpha\\sim\\mathrm{Uniform}(0,1)$, annealed temperature $\\tau$) by minimizing, per class, a one-step gradient-matching loss $D(\\nabla_\\theta \\ell(f_{\\theta_0} (\\mathbf{A}'(\\Omega),\\mathbf{X}'),\\mathcal{Y}'), \\nabla_\\theta \\ell(f_{\\theta_0}(\\mathcal{T}),\\mathcal{Y}))$ averaged over $K_1{=}1000$ sampled random initializations $\\theta_0$, plus a sparsity regularizer on $\\sigma(\\Omega)$ and (for sum-pooling GNNs only) a second regularizer derived from the generalization bound. A 3-layer GCN is the network in the loop; labels $\\mathcal{Y}'$ are fixed hard labels. For node classification the same one-step loss is substituted directly into GCond's framework.", "evidence": "Table 1 (graph classification, GCN evaluator, hard labels): ogbg-molbace at 2 graphs/0.2% budget reaches 0.657 ROC-AUC vs. 0.714 whole-dataset (92% retained); ogbg-molhiv at 0.3% budget reaches 0.731 vs. 0.757 whole (96.5% retained); DD dataset at 0.2%/2.1%/10.6% budgets retains 89%/93%/98% of whole-dataset accuracy. Table 3 (node classification, ported into GCond's protocol): DosCond matches GCond within ~2 points on Cora/Citeseer/Pubmed/Arxiv/Flickr while running up to 40x faster per 100 epochs (e.g. Cora 3.5s vs. 75.9s). Table 2: on CIFAR10 (50 graphs/class) DosCond takes ~5 minutes to reach the accuracy DosCond-Bi (the bi-level ablation) takes 75 minutes to reach — 6.7% of the training cost. Key ablation: DCG (a DC variant that only learns features on randomly-selected, i.e. non-learned, structure) trails DosCond by a large margin on graph classification, showing that learning discrete structure jointly with features (not just features) is necessary to preserve classification-relevant information.", "limitations": "Performance on CIFAR10-as-graph is comparatively weak, attributed to its more complex topology needing more synthetic parameters than the tested budgets provide; datasets are still small-to-modest scale (molecular graphs, TU datasets, one 1,109-graph e-commerce set) with no ImageNet-scale or web-scale graph tested; the one-step approximation is justified by a bound that assumes the largest gradient gap occurs at initialization, an assumption not directly verified beyond the two loss-term-scale plots; and the paper's own future work flags a lack of interpretability for the learned discrete structures." } }, { "id": "rtp-2022", "type": "paper", "title": "Remember the Past: Distilling Datasets into Addressable Memories for Neural Networks", "short": "Addressable memories", "arxiv": "2206.02916", "url": null, "date": "2022-06-06", "venue": { "name": "NeurIPS 2022", "status": "conference" }, "authors": [ "Zhiwei Deng", "Olga Russakovsky" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization" ], "facets": { "parameterization": [ "addressable-memory" ], "labels": "hard", "prior": [ "none" ], "regime": "bilevel", "scale": [ "mnist-cifar", "tiny-imagenet" ] }, "applications": [ "continual-learning" ], "tier": "notable", "links": { "code": "https://github.com/princetonvisualai/RememberThePast-DatasetDistillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "builds_on": [ "dd-2018", "dc-2021", "dsa-2021" ], "explained": { "before": "Existing distillation methods (dd-2018's BPTT, dc-2021/dsa-2021's gradient matching, kip-2021's kernel ridge regression) all assign each class its own independent set of learnable images, so storage grows linearly with the number of classes and any information shared between classes (e.g. related textures across visually similar categories) is re-learned redundantly in each class's images.", "problem": "A per-class, independent-sample representation cannot exploit cross-class redundancy, does not generalize to continuous or unseen label spaces, and its previously reported weakness -- that back-propagation through time (BPTT) underperforms single-step gradient matching -- is, the paper argues, a property of how BPTT has been used (short unrolls, no momentum) rather than of BPTT itself.", "idea": "Reformulate distillation as memory addressing: store one shared pool of basis vectors (\"memories\") plus learned per-query addressing matrices that linearly recombine bases into class-conditioned synthetic images, so the memory pool can be reused across classes and the compressed size decouples from the number of classes; separately, show that BPTT itself is a strong distillation algorithm once trained with momentum and long (100-200 step) unrolled inner loops.", "method": "A set of K basis vectors $\\{b_1,...,b_K\\}$ (each shaped like an image, optionally stored at half resolution and bilinearly upsampled) forms a shared memory $\\mathcal{M}$. For a label vector $y$, r addressing matrices $\\{A_1,...,A_r\\}$ (one-hot or continuous) produce synthetic images $x_i'^T = y^T A_i [b_1;...;b_K]^T$. Both memories and addressing matrices are trained end-to-end by back-propagation through time: an inner loop trains a randomly initialized network from scratch on the addressed synthetic set for T steps (with momentum, T up to 200) via SGD, and the outer loop backpropagates a generalization loss on a real minibatch through the whole inner trajectory to update the memories and addressing matrices. Storage budget is counted as total parameter size: size(bases)+size(addressing matrices) is constrained to approximately match N*C*size(image) for a baseline of N images per C classes, i.e. an explicit equal-storage protocol against pixel-space baselines. Hard, discrete class labels.", "evidence": "CIFAR-10, 1 image/class-equivalent storage, ConvNet: 66.4% vs. KIP 49.9%, MTT 46.3%, DM 26.0% (+16.5pp over the next best, Table 1). SVHN, same budget: 87.3% vs. KIP 57.3% (+30pp). CIFAR-100, 10 image/class budget: 42.9% vs. TM 40.1%, DM 29.7%. TinyImageNet, 1 image/class: 16.0% vs. TM 8.8%, DM 3.9%. Ablation (Table 2): single-step gradient matching 28.8% (CIFAR-10, 1 img/class) -> vanilla BPTT (no downsampling, no memory addressing) 49.1% -> + downsampling 55.2% -> + memory-addressing formulation 64.2% -> + data augmentation (full) 66.4%, isolating memory addressing's contribution at about 9pp on top of downsampling and about 36pp on top of single-step gradient matching. Momentum/unroll-length analysis (Fig. 4): adding momentum gains 7.0-9.2pp on CIFAR-10 at 10-100 inner steps; lengthening inner unrolls from 1 to 100+ steps gains 18.2pp (CIFAR-10) and 42.3pp (SVHN). Continual learning \"compress-then-recall\" (Table 3): 74.07% RA on MANY vs. best prior sp-La 50.81% (+23.2pp). Information sharing is visualized directly (Fig. 4): cosine similarity of addressing coefficients between CIFAR-100 classes shows visually similar classes (maple/oak/palm/pine/willow) share memory usage while dissimilar ones (lawn mower/rocket) do not.", "limitations": "States that the costly inner-loop optimization (100-200-step unrolled BPTT per outer step) may become time-consuming on larger models or datasets, and suggests this could be addressed by pairing the memory formulation with a different (cheaper) learning framework -- i.e. the method as presented does not scale to ImageNet-size settings, only up to TinyImageNet. No wall-clock or GPU-hour figures are reported for synthesis cost, only \"one GPU per experiment run.\" The equal-storage protocol (matching total parameter count) is a reasonable but not universally adopted accounting convention, and later parameterization papers (fred-2023, ddif-2025) use different conventions, complicating cross-paper comparison. The paper also flags a general societal risk that distilled data may under-represent minority populations, arguing its own higher accuracy is a partial mitigation rather than a solution." }, "summary": "Compresses a dataset into compact addressable memories -- a set of bases ('memories') shared across classes and combined through learned addressing functions to generate diverse training examples -- so compressed size need not grow linearly with the number of classes, and generalized queries beyond the original classes become possible; reports up to 16.5% and 9.7% retained-accuracy gains on CIFAR-10/100 respectively and, applied to continual learning, a 23.2% accuracy improvement on the MANY benchmark.", "abstract": "We propose an algorithm that compresses the critical information of a large dataset into compact addressable memories. These memories can then be recalled to quickly re-train a neural network and recover the performance (instead of storing and re-training on the full original dataset). Building upon the dataset distillation framework, we make a key observation that a shared common representation allows for more efficient and effective distillation. Concretely, we learn a set of bases (aka ``memories'') which are shared between classes and combined through learned flexible addressing functions to generate a diverse set of training examples. This leads to several benefits: 1) the size of compressed data does not necessarily grow linearly with the number of classes; 2) an overall higher compression rate with more effective distillation is achieved; and 3) more generalized queries are allowed beyond recalling the original classes. We demonstrate state-of-the-art results on the dataset distillation task across six benchmarks, including up to 16.5% and 9.7% in retained accuracy improvement when distilling CIFAR10 and CIFAR100 respectively. We then leverage our framework to perform continual learning, achieving state-of-the-art results on four benchmarks, with 23.2% accuracy improvement on MANY. The code is released on our project webpage https://github.com/princetonvisualai/RememberThePast-DatasetDistillation.", "awesome_sections": [ "Main / Distilled Dataset Parametrization" ] }, { "id": "distill-cf-2022", "type": "paper", "title": "Infinite Recommendation Networks: A Data-Centric Approach", "short": "Distill-CF", "arxiv": "2206.02626", "url": null, "date": "2022-06-03", "venue": { "name": "NeurIPS 2022", "status": "conference" }, "authors": [ "Noveen Sachdeva", "Mehak Preet Dhaliwal", "Carole-Jean Wu", "Julian McAuley" ], "kind": "method", "settings": [ "other-data" ], "families": [ "kernel-closed-form" ], "facets": { "regime": "single-level", "prior": [ "ntk-kernel" ] }, "tier": "notable", "links": { "code": "https://github.com/noveens/distill_cf", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Uses the Neural Tangent Kernel equivalence of an infinitely-wide autoencoder (infinity-AE) to give recommendation-model training a closed-form solution, then builds Distill-CF on top of it with differentiable Gumbel-sampling to synthesize tiny, high-fidelity summaries of large, sparse user-item interaction matrices; reports 96-105% of infinity-AE's full-data performance using as little as 0.1% of the original interactions.", "abstract": "We leverage the Neural Tangent Kernel and its equivalence to training infinitely-wide neural networks to devise $\\infty$-AE: an autoencoder with infinitely-wide bottleneck layers. The outcome is a highly expressive yet simplistic recommendation model with a single hyper-parameter and a closed-form solution. Leveraging $\\infty$-AE's simplicity, we also develop Distill-CF for synthesizing tiny, high-fidelity data summaries which distill the most important knowledge from the extremely large and sparse user-item interaction matrix for efficient and accurate subsequent data-usage like model training, inference, architecture search, etc. This takes a data-centric approach to recommendation, where we aim to improve the quality of logged user-feedback data for subsequent modeling, independent of the learning algorithm. We particularly utilize the concept of differentiable Gumbel-sampling to handle the inherent data heterogeneity, sparsity, and semi-structuredness, while being scalable to datasets with hundreds of millions of user-item interactions. Both of our proposed approaches significantly outperform their respective state-of-the-art and when used together, we observe 96-105% of $\\infty$-AE's performance on the full dataset with as little as 0.1% of the original dataset size, leading us to explore the counter-intuitive question: Is more data what you need for better recommendation?", "awesome_sections": [ "Applications / Recommender Systems" ], "notes": "" }, { "id": "privacy-for-free-2022", "type": "paper", "title": "Privacy for Free: How does Dataset Condensation Help Privacy?", "short": "Privacy for Free", "arxiv": "2206.00240", "url": null, "date": "2022-06-01", "venue": { "name": "ICML 2022", "status": "conference" }, "authors": [ "Tian Dong", "Bo Zhao", "Lingjuan Lyu" ], "kind": "trustworthy", "settings": [ "image-classification" ], "families": [], "applications": [ "privacy" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Argues dataset condensation, though designed for training efficiency, also serves as a private-data generator \"for free,\" and proves for linear (then extended to non-linear) feature extractors that any one of the n raw samples has only O(m/n) influence on the parameter distribution of networks trained on m condensed samples. Empirically supports this with loss-based and likelihood-based membership-inference attacks against DC-synthesized data. This claim's empirical and theoretical basis is disputed by no-free-lunch-privacy-2022.", "abstract": "To prevent unintentional data leakage, research community has resorted to data generators that can produce differentially private data for model training. However, for the sake of the data privacy, existing solutions suffer from either expensive training cost or poor generalization performance. Therefore, we raise the question whether training efficiency and privacy can be achieved simultaneously. In this work, we for the first time identify that dataset condensation (DC) which is originally designed for improving training efficiency is also a better solution to replace the traditional data generators for private data generation, thus providing privacy for free. To demonstrate the privacy benefit of DC, we build a connection between DC and differential privacy, and theoretically prove on linear feature extractors (and then extended to non-linear feature extractors) that the existence of one sample has limited impact ($O(m/n)$) on the parameter distribution of networks trained on $m$ samples synthesized from $n (n \\gg m)$ raw samples by DC. We also empirically validate the visual privacy and membership privacy of DC-synthesized data by launching both the loss-based and the state-of-the-art likelihood-based membership inference attacks. We envision this work as a milestone for data-efficient and privacy-preserving machine learning.", "awesome_sections": [ "Applications / Privacy" ], "notes": "families left empty: the theoretical/empirical privacy analysis applies to dataset condensation generally, not one mechanism family. Disputed by no-free-lunch-privacy-2022 (see handoff notes)." }, { "id": "frepo-2022", "type": "paper", "title": "Dataset Distillation using Neural Feature Regression", "short": "FRePo", "arxiv": "2206.00719", "url": null, "date": "2022-06-01", "venue": { "name": "NeurIPS 2022", "status": "conference" }, "authors": [ "Yongchao Zhou", "Ehsan Nezhadarya", "Jimmy Ba" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "kernel-closed-form" ], "facets": { "parameterization": [ "pixels" ], "labels": "mixed", "prior": [ "random-networks" ], "regime": "bilevel", "scale": [ "tiny-imagenet", "imagenet-subsets", "imagenet-1k" ] }, "tier": "core", "links": { "code": "https://github.com/yongchao97/FRePo", "project": "https://sites.google.com/view/frepo" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Replaces the closed-form kernel-ridge-regression solver of prior kernel-based distillation with regression onto the features of a pool of finite-width networks (neural Feature Regression with Pooling), reducing memory by an order of magnitude and training time by two orders of magnitude while improving accuracy on CIFAR-100, Tiny-ImageNet and ImageNet-1K.", "abstract": "Dataset distillation aims to learn a small synthetic dataset that preserves most of the information from the original dataset. Dataset distillation can be formulated as a bi-level meta-learning problem where the outer loop optimizes the meta-dataset and the inner loop trains a model on the distilled data. Meta-gradient computation is one of the key challenges in this formulation, as differentiating through the inner loop learning procedure introduces significant computation and memory costs. In this paper, we address these challenges using neural Feature Regression with Pooling (FRePo), achieving the state-of-the-art performance with an order of magnitude less memory requirement and two orders of magnitude faster training than previous methods. The proposed algorithm is analogous to truncated backpropagation through time with a pool of models to alleviate various types of overfitting in dataset distillation. FRePo significantly outperforms the previous methods on CIFAR100, Tiny ImageNet, and ImageNet-1K. Furthermore, we show that high-quality distilled data can greatly improve various downstream applications, such as continual learning and membership inference defense. Please check out our webpage at https://sites.google.com/view/frepo.", "awesome_sections": [ "Main / Kernel-Based Distillation" ], "builds_on": [ "kip-2021" ], "explained": { "before": "Responds directly to kip-2021's exact-NTK closed-form inner solver, which the paper states requires \"thousands of GPU hours and sophisticated implementation of the distributed kernel computation framework\" for modern architectures, and to dd-2018-style unrolled BPTT, which needs the whole training trajectory in memory. Also responds to surrogate-matching alternatives (dc-2021, dsa-2021 gradient matching; CAFE/DM distribution matching; mtt-2022 trajectory matching), arguing each surrogate can introduce its own bias and typically overfits to a single learning algorithm, training stage, or architecture.", "problem": "Meta-gradient computation for dataset distillation is expensive (unrolled BPTT) or requires an intractable exact kernel (KIP-style NTK for deep nets); separately, distilled data tends to overfit to the specific algorithm, training stage, or architecture used during synthesis, hurting cross-architecture transfer.", "method": "Splits the matching network into a feature extractor $f(\\cdot,\\theta)$ and a linear head, and instead of learning the head by gradient descent, solves for it via kernel ridge regression on the extractor's own features (a \"conjugate kernel,\" not the NTK): $\\mathcal{L}=\\tfrac12\\|Y_t-K^\\theta_{X_tX_s}(K^\\theta_{X_sX_s}+\\lambda I)^{-1}Y_s\\|_2^2$. Backpropagating this loss through the fixed feature extractor gives the meta-gradient for the synthetic images $X_s$ (and, since the loss is smooth in $Y_s$, learned labels), with no need to unroll the extractor's own training. A \"model pool\" of $m$ feature extractors at different initializations and training stages is maintained; each step samples one pool member, uses it for the KRR meta-gradient, takes one online training step on it with the current synthetic data, and reinitializes it once it has been updated more than $K$ steps -- diversifying which extractor is matched against, unlike a single periodically-reset model. Stored artifact: only the synthetic images and (optionally) learned labels; the pool is discarded after training.", "evidence": "CIFAR-100 x 1 IPC x FRePo x learned labels x 28.7% (Table 1) vs. MTT 24.3%, KIP 15.7%, DSA 16.8%, DM 12.2%; Tiny-ImageNet x 1 IPC: 15.4% (FRePo) vs. MTT 8.8%, DSA 6.6%; CUB-200 x 1 IPC: 12.4% (FRePo) vs. MTT 2.2% -- the paper's headline claim that FRePo is exceptionally strong on complex, many-class label spaces at 1 IPC. Cross-architecture transfer (Table 2, CIFAR-10 x 10 IPC, trained on Conv-BN): FRePo beats DSA/DM/MTT/KIP on every unseen evaluation architecture (Conv-NN 65.5% vs. MTT's 41.6%; AlexNet 61.9% vs. KIP's 57.2%; VGG-BN 59.4% vs. MTT's 46.6%), and the paper notes instance normalization -- used by DSA/DM/MTT -- specifically hurts their cross-architecture transfer, while FRePo's images \"look natural\" and carry less architectural bias. Resized ImageNet-1K (64x64): 1-2 IPC (1k/2k images) reaches 7.5%/9.7% top-1 vs. 1.1%/1.4% for an equally sized random real subset (no comparison to MTT is given at this scale in-paper). Cost: on CIFAR-100 at 1 IPC, FRePo matches MTT's accuracy (~24%) in 38 seconds vs. MTT's 3,805 seconds (~100x faster) and reaches 92% of its own final accuracy in 385 seconds, with roughly 10x lower peak GPU memory (Fig. 3). Downstream: 5-/10-step class-incremental continual learning on CIFAR-100 (41.6%/38.0% final accuracy vs. DM's 33.9%/34.0%) and membership- inference defense (attack AUC near 0.5, i.e., near random guessing, on MNIST/FashionMNIST at 500 images) both improve over DSA/DM at matched budget. Ablation: label learning is unnecessary on simple, few-class datasets (MNIST) but becomes crucial on complex many-class datasets (CIFAR-100, Tiny-ImageNet), isolating label richness as the mechanism behind the many-class gains.", "limitations": "Requires a wider network architecture than prior methods for the KRR component to behave well (a narrow feature dimension causes a significant accuracy drop, per Appendix C.6), so comparisons are not always at matched architecture width. No data augmentation is used during synthesis (only at evaluation), unlike dsa-2021/mtt-2022. Diversifying the model pool with multiple architectures (rather than just different seeds/stages of one architecture) was tried but found unstable, since meta-gradients from different architectures can disagree sharply -- so the reported cross-architecture generalization comes from a single-architecture pool, not explicit multi-architecture training. Splitting per-class distillation across sequential continual-learning steps was observed to underperform distilling the same total budget in one pass, suggesting the joint, whole-dataset view is important. ImageNet-1K results are at only 1-2 IPC and use hard evaluation without any teacher-relabeling or soft-label scheme (unlike tesla-2023's contemporaneous ImageNet-1K soft-label approach), leaving unclear how far the closed-form-KRR proxy scales past very low IPC on datasets with 1,000 classes." } }, { "id": "idc-2022", "type": "paper", "title": "Dataset Condensation via Efficient Synthetic-Data Parameterization", "short": "IDC", "arxiv": "2205.14959", "url": null, "date": "2022-05-30", "venue": { "name": "ICML 2022", "status": "conference" }, "authors": [ "Jang-Hyun Kim", "Jinuk Kim", "Seong Joon Oh", "Sangdoo Yun", "Hwanjun Song", "Joonhyun Jeong", "Jung-Woo Ha", "Hyun Oh Song" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "parameterization" ], "tier": "core", "links": { "code": "https://github.com/snu-mllab/efficient-dataset-condensation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "dc-2021 and dsa-2021 match network gradients between real and synthetic batches inside a bi-level loop, training the matching networks on the synthetic data itself; dm-2023's feature matching is faster but reported to underperform gradient matching. All prior methods store condensed data pixel-for-pixel at the original resolution, one image per stored unit.", "problem": "Under a fixed storage budget, pixel-for-pixel storage does not exploit the spatial regularity of natural data (nearby pixels are correlated), so the number of synthetic samples the budget can hold is unnecessarily small; separately, gradient matching against networks trained on the tiny synthetic set causes those networks to overfit early, so the real/synthetic gradient norms diverge and matching quality (measured with MSE-style objectives) degrades during optimization.", "idea": "Store each condensed unit at reduced resolution and deterministically upsample it into several images at matching time (multi-formation), so a fixed pixel budget yields more, regularized training images; combine this with an improved gradient-matching recipe that computes gradients from networks trained on the real data (not the synthetic data), matched by a distance-based (MSE) rather than cosine objective, with stronger network regularization.", "method": "A multi-formation function f partitions each stored image into equal tiles and bilinearly upsamples each tile back to full resolution (factor 2 for CIFAR-10/Speech Commands, factor 3 for ImageNet), turning n stored elements into n' > n training images under the same total pixel budget; f is differentiable, so synthetic pixels are optimized end-to-end through it. Gradients for matching are computed from a network theta_T periodically re-initialized and trained a few epochs on the real data (not the synthetic data), using CutMix and augmentation for stronger regularization, with an MSE (not cosine) distance between per-class real and synthetic gradients. Hard labels throughout; no learned labels or teacher.", "evidence": "CIFAR-10, 50 images/class budget (equal pixel storage), ConvNet-3, hard labels: IDC 74.5% vs. DSA 60.6%, DM 65.6%, and IDC-I (same optimization, no multi-formation) 69.5% (Table 1) -- isolating multi-formation's contribution at ~5pp on top of the optimization fix's ~9pp gain over DM. ImageNet-10, 20 images/class, ResNetAP-10: IDC 76.6% vs. DM 59.3%, IDC-I 65.5% (Table 3). Speech Commands, 20 spectrograms/class: IDC 86.6% vs. DM 77.2%, IDC-I 83.0% (Table 4). Storage accounting is total pixels (equal-storage comparison with pixel-space DSA/DM/KIP baselines): e.g. \"50x32x32\" pixels/class is the same budget whether stored as 50 full images or 200 half-resolution tiles. Ablation (Table 5, CIFAR-10, IPC=50): DSA (Syn-trained networks + cosine) 60.6% -> Real-trained networks + cosine 63.4% -> Real-trained + MSE 67.0% -> + regularization 69.5% (this is IDC-I; multi-formation is not yet added), separating the optimization fix from the parameterization gain. Ablation on multi-formation factor (Table 7): factor 2 is optimal for CIFAR-10 (74.5%) but factor 3-4 is optimal for ImageNet-10 (76.6/77.5%), i.e. the ideal number-of-images- vs-resolution trade-off is dataset- and resolution-dependent. Post-downsampling comparison (Table 6): IDC's end-to-end optimized multi-formation (74.5% at 50x32x32 px) beats naively downsampling then upsampling a 4x larger stored set (68.8% at 200x16x16 px), showing the gain is from co-optimizing storage with the formation function, not just from resolution reduction itself.", "limitations": "IDC requires more training steps to converge than baselines because of the formation process, though a fixed-training-step comparison (Table 2) still favors IDC. Optimal multi-formation factor is dataset- and resolution-dependent and must be tuned (2 for CIFAR-10, 3-4 for ImageNet), with no automatic selection rule. No GPU-hour or wall-clock synthesis cost is reported (only per-step evaluation training time in Table 2, which measures downstream training, not condensation cost). The matching objective is gradient matching, not distribution matching; the paper explicitly discusses (Section \"Discussion on Dataset Structure\") that deviating from same-shape storage toward dictionary codes or generator parameterizations was considered but not pursued because such structures can require more storage or compute than they save." }, "facets": { "parameterization": [ "downsampled-multiformation" ], "labels": "hard", "prior": [ "trained-experts" ], "regime": "single-level", "scale": [ "mnist-cifar", "imagenet-subsets", "large-scale-other" ] }, "builds_on": [ "dc-2021", "dsa-2021" ], "summary": "Proposes an efficient synthetic-data parameterization that packs multiple down-sampled or augmented image formations into the storage of one, exploiting spatial data regularity, together with an improved optimization for gradient-matching-based condensation; substantially improves condensed-data quality over the prior state of the art on CIFAR-10, ImageNet subsets, and Speech Commands.", "abstract": "The great success of machine learning with massive amounts of data comes at a price of huge computation costs and storage for training and tuning. Recent studies on dataset condensation attempt to reduce the dependence on such massive data by synthesizing a compact training dataset. However, the existing approaches have fundamental limitations in optimization due to the limited representability of synthetic datasets without considering any data regularity characteristics. To this end, we propose a novel condensation framework that generates multiple synthetic data with a limited storage budget via efficient parameterization considering data regularity. We further analyze the shortcomings of the existing gradient matching-based condensation methods and develop an effective optimization technique for improving the condensation of training data information. We propose a unified algorithm that drastically improves the quality of condensed data against the current state-of-the-art on CIFAR-10, ImageNet, and Speech Commands.", "awesome_sections": [ "Main / Distilled Dataset Parametrization" ] }, { "id": "deepcore-2022", "type": "paper", "title": "DeepCore: A Comprehensive Library for Coreset Selection in Deep Learning", "short": "DeepCore", "arxiv": "2204.08499", "url": null, "date": "2022-04-18", "venue": { "name": "DEXA 2022", "status": "conference" }, "authors": [ "Chengcheng Guo", "Bo Zhao", "Yanbing Bai" ], "kind": "baseline", "settings": [ "image-classification" ], "families": [ "coreset-selection" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Contributes a code library unifying coreset-selection methods for deep learning and an empirical study of them on CIFAR-10 and ImageNet under matched settings. Reports that although individual methods have advantages in specific settings, random selection remains a strong baseline overall.", "abstract": "Coreset selection, which aims to select a subset of the most informative training samples, is a long-standing learning problem that can benefit many downstream tasks such as data-efficient learning, continual learning, neural architecture search, active learning, etc. However, many existing coreset selection methods are not designed for deep learning, which may have high complexity and poor generalization performance. In addition, the recently proposed methods are evaluated on models, datasets, and settings of different complexities. To advance the research of coreset selection in deep learning, we contribute a comprehensive code library, namely DeepCore, and provide an empirical study on popular coreset selection methods on CIFAR10 and ImageNet datasets. Extensive experiments on CIFAR10 and ImageNet datasets verify that, although various methods have advantages in certain experiment settings, random selection is still a strong baseline.", "notes": "" }, { "id": "it-gan-2022", "type": "paper", "title": "Synthesizing Informative Training Samples with GAN", "short": "IT-GAN", "arxiv": "2204.07513", "url": null, "date": "2022-04-15", "venue": { "name": "NeurIPS 2022 Workshop", "status": "workshop" }, "authors": [ "Bo Zhao", "Hakan Bilen" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "generative-latent-optimization" ], "facets": { "parameterization": [ "generative-latent" ], "labels": "hard", "prior": [ "gan" ], "regime": "single-level", "scale": [ "mnist-cifar" ] }, "tier": "core", "links": { "code": "https://github.com/vico-uoe/it-gan", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Freezes a pretrained GAN and optimizes the latent vectors that generate informative training images, rather than optimizing for visual realism, so synthesized samples are chosen to accelerate and improve downstream network training; verified to let networks train faster and reach better performance than with plain GAN samples, offering a path to dataset condensation without directly optimizing pixels.", "abstract": "Remarkable progress has been achieved in synthesizing photo-realistic images with generative adversarial networks (GANs). Recently, GANs are utilized as the training sample generator when obtaining or storing real training data is expensive even infeasible. However, traditional GANs generated images are not as informative as the real training samples when being used to train deep neural networks. In this paper, we propose a novel method to synthesize Informative Training samples with GAN (IT-GAN). Specifically, we freeze a pre-trained GAN model and learn the informative latent vectors that correspond to informative training samples. The synthesized images are required to preserve information for training deep neural networks rather than visual reality or fidelity. Experiments verify that the deep neural networks can learn faster and achieve better performance when being trained with our IT-GAN generated images. We also show that our method is a promising solution to dataset condensation problem.", "explained": { "before": "Bilevel meta-learning distillation (DD, GTN) and the then-new single-level matching methods DC and DSA and DM (Zhao & Bilen) all optimize pixels directly; separately, GAN-based data generators (BigGAN) and GAN inversion had been used to make training-sample generators, but only to look realistic, not to be informative.", "problem": "A BigGAN trained on CIFAR-10/100 and sampled normally, or inverted to reconstruct real images pixel-for-pixel, produces images that look realistic but train a ResNet18 far worse than real data (77.8% and 82.9% vs. 93.4% on CIFAR-10): realism and informativeness for training are different objectives, and GAN inversion's pixel/feature reconstruction loss does not target the second one.", "idea": "Freeze a pretrained GAN and, instead of optimizing pixels or matching a discriminator, optimize the GAN's latent vectors directly with a dataset-condensation matching loss (distribution matching, following DM), so the generator becomes a vehicle for informative rather than realistic samples.", "method": "A BigGAN is pretrained per dataset (CIFAR-10, CIFAR-100) with differentiable-augmentation GAN training and then frozen. One latent vector $\\bm{z}_i$ is kept per real training image, initialized by GAN inversion ($\\arg\\min_z \\|\\psi(G(z))-\\psi(x)\\|^2 + \\lambda_{pixel}\\|G(z)-x\\|^2$) so the whole latent set $\\mathcal{Z}$ has the same cardinality as $\\mathcal{T}$. $\\mathcal{Z}$ is then optimized (BigGAN still frozen) to minimize a class-wise distribution-matching loss $\\mathcal{L}_{con}$ between real and generated features under randomly sampled embedding networks $\\psi_{\\vartheta}$ and differentiable Siamese augmentation, plus a per-sample regularizer $R$ that keeps a generated image's features close to its paired real image's features so that independently-trained latent batches stay diverse rather than homogenizing. Because $|\\mathcal{Z}|\\approx|\\mathcal{T}|$, the set is split into batches (e.g. 1250/class for CIFAR-10, 500/class for CIFAR-100) trained independently and then pooled (\"latent vector ensemble\"). At evaluation, latent vectors are decoded through the frozen BigGAN to produce images with their original hard labels, and a fresh network is trained on these generated images from scratch with SGD.", "evidence": "Whole-set learning (CIFAR-10/100, using the full learned latent set, ResNet18, hard labels, Table 1): GAN 77.8%/45.2%, GAN Inversion 82.9%/55.0%, IT-GAN 85.7%/60.1%, real-data upper bound 93.4%/74.1%. Against DM at matched 25%-of-dataset storage budget (Table 2, hard labels): CIFAR-10 ConvNet 82.8% vs. DM 80.8% (upper bound 84.8%); CIFAR-10 ResNet18 85.7% vs. DM 85.1%; CIFAR-100 ConvNet 55.7% vs. DM 50.5% (upper bound 56.2%); CIFAR-100 ResNet18 60.1% vs. DM 56.7%. Cross-architecture (Table 3, latents learned with ConvNet only): 82.8-86.0% on CIFAR-10 and 55.7-60.4% on CIFAR-100 across ConvNet, VGG19, ResNet18, WRN-16-8, MobileNetV2, i.e. no architecture-specific collapse. Key ablation (Table T6, appendix): with latent batches mixed/re-split randomly every iteration, accuracy without the regularizer $R$ ($\\lambda=0$) is 69.7% vs. 71.3% with tuned $\\lambda$, confirming $R$ is what prevents independently-optimized latent batches from becoming redundant copies of each other. No GPU-hour or wall-clock cost is reported; storage cost is quantified instead (40.8MB total: 16.4MB frozen BigGAN + 24.4MB of 128-d latents, vs. 162MB raw CIFAR).", "limitations": "Tested only at CIFAR resolution (32x32) and only at a very large \"25% of dataset\" budget (roughly 1000+ latents/class), not the low-IPC regime (1/10/50 images per class) that dominates the field's tables, so it is unclear how IT-GAN behaves at small budgets; the generator is a BigGAN trained from scratch on the exact target dataset (CIFAR-10 or CIFAR-100), so none of the reported gain is attributable to a generator pretrained on other data, and the paper does not test transferring one dataset's GAN to another; evaluation uses only hard labels with no teacher relabeling; the cost of pretraining and storing a BigGAN per dataset, and of pretraining hundreds of ConvNets to serve as embedding functions, is not counted against the reported storage savings." }, "builds_on": [ "dm-2023", "dc-2021", "dsa-2021" ], "awesome_sections": [ "Main / Generative Distillation / GAN" ] }, { "id": "fedsynth-2022", "type": "paper", "title": "FedSynth: Gradient Compression via Synthetic Data in Federated Learning", "short": "FedSynth", "arxiv": "2204.01273", "url": null, "date": "2022-04-04", "venue": { "name": "arXiv 2022", "status": "preprint" }, "authors": [ "Shengyuan Hu", "Jack Goetz", "Kshitiz Malik", "Hongyuan Zhan", "Zhe Liu", "Yue Liu" ], "kind": "application", "settings": [ "image-classification" ], "families": [], "applications": [ "federated-learning" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Instead of transmitting a sparsified model update, each client learns and uploads a small synthetic dataset such that training on it reproduces the client's local update, which the server recovers and aggregates as usual. Reports comparable or better upstream compression than random-masking baselines on three common federated learning benchmarks.", "abstract": "Model compression is important in federated learning (FL) with large models to reduce communication cost. Prior works have been focusing on sparsification based compression that could desparately affect the global model accuracy. In this work, we propose a new scheme for upstream communication where instead of transmitting the model update, each client learns and transmits a light-weight synthetic dataset such that using it as the training data, the model performs similarly well on the real training data. The server will recover the local model update via the synthetic data and apply standard aggregation. We then provide a new algorithm FedSynth to learn the synthetic data locally. Empirically, we find our method is comparable/better than random masking baselines in all three common federated learning benchmark datasets.", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "families left empty: abstract does not name the matching mechanism used to learn the synthetic data." }, { "id": "mtt-2022", "type": "paper", "title": "Dataset Distillation by Matching Training Trajectories", "short": "MTT", "arxiv": "2203.11932", "url": null, "date": "2022-03-22", "venue": { "name": "CVPR 2022", "status": "conference" }, "authors": [ "George Cazenavette", "Tongzhou Wang", "Antonio Torralba", "Alexei A. Efros", "Jun-Yan Zhu" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "trajectory-matching" ], "facets": { "parameterization": [ "pixels" ], "labels": "hard", "prior": [ "trained-experts" ], "regime": "single-level", "scale": [ "tiny-imagenet", "imagenet-subsets" ] }, "tier": "landmark", "links": { "code": "https://github.com/georgecazenavette/mtt-distillation", "project": "https://georgecazenavette.github.io/mtt-distillation/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Introduces trajectory matching: instead of matching single-step gradients, optimize synthetic data so that training a network on it for several steps lands close to where a network trained on real data lands after many more steps, using precomputed, cached expert training trajectories to avoid recomputing them, and shows this outperforms prior methods and scales to higher-resolution images.", "abstract": "Dataset distillation is the task of synthesizing a small dataset such that a model trained on the synthetic set will match the test accuracy of the model trained on the full dataset. In this paper, we propose a new formulation that optimizes our distilled data to guide networks to a similar state as those trained on real data across many training steps. Given a network, we train it for several iterations on our distilled data and optimize the distilled data with respect to the distance between the synthetically trained parameters and the parameters trained on real data. To efficiently obtain the initial and target network parameters for large-scale datasets, we pre-compute and store training trajectories of expert networks trained on the real dataset. Our method handily outperforms existing methods and also allows us to distill higher-resolution visual data.", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ], "explained": { "before": "Responds to dd-2018's expensive, unstable full backprop-through-training and to dc-2021/dsa-2021's single-step gradient matching, which the paper argues drifts from the true training trajectory because it only matches one local gradient direction rather than where training actually ends up many steps later; also compares against kip-2021's infinite-width kernel approach.", "problem": "Single-step gradient matching is a short-horizon proxy: matching one step's gradient direction does not guarantee the resulting long trajectory of parameters resembles real training, so accumulated drift over many steps caps accuracy. Full bi-level BPTT (dd-2018) would fix this but is too memory-hungry and unstable to unroll over realistic numbers of steps.", "idea": "Precompute many \"expert\" trajectories from ordinary training runs on real data, then optimize the synthetic images so that a few steps of training on them move a network's parameters to (approximately) the same place an expert's parameters reach many steps later, starting from the same point on the expert trajectory -- matching long-range parameter movement instead of single-step gradients or the full trajectory.", "method": "Optimizes a per-class synthetic image set (pixels) plus a learned synthetic-step learning rate. In the loop: expert trajectories $\\{\\theta^*_t\\}$ are trained once on real data and cached to disk (not part of the stored distilled artifact). Each distillation step samples a start point $\\theta^*_t$ from a random cached expert and epoch, initializes a student at $\\hat\\theta_t=\\theta^*_t$, takes $N$ differentiable-augmentation SGD steps on the synthetic images to reach $\\hat\\theta_{t+N}$, and minimizes the normalized squared distance $\\mathcal{L}=\\|\\hat\\theta_{t+N}-\\theta^*_{t+M}\\|_2^2 / \\|\\theta^*_t-\\theta^*_{t+M}\\|_2^2$ to the expert's parameters $M>N$ real-training epochs later (Eq. 2), backpropagating through the $N$ student steps into the images. A per-class batching scheme (Eq. 3) samples a fresh mini-batch of synthetic images at every inner step to control memory when the synthetic set is large. Labels are fixed/hard.", "evidence": "Table 1 (ConvNet-128, ZCA whitening, hard labels): CIFAR-10 IPC1 46.3+/-0.8%, IPC10 65.3+/-0.7%, IPC50 71.6+/-0.2% vs. full-data 84.8+/-0.1%; CIFAR-100 IPC1 24.3+/-0.3%, IPC10 40.1+/-0.4%, IPC50 47.7+/-0.2% vs. full-data 56.2+/-0.3%; Tiny-ImageNet (64x64) IPC1 8.8+/-0.3%, IPC10 23.2+/-0.2%, IPC50 28.0+/-0.3% vs. full-data 37.6+/-0.4%, beating DC/DSA/DM/CAFE/CAFE+DSA at every matched IPC. Table 2 vs. kip-2021 on the same 128-width ConvNet: CIFAR-10 IPC1 46.3% vs. KIP 38.3% (49.9% on a 1024-width net), IPC10 65.3% vs. 57.6%, IPC50 71.5% vs. 65.8%; CIFAR-100 IPC1 24.3% vs. 18.2%, IPC10 39.4% vs. 32.8%. Table 4, 128x128 ImageNet subsets, IPC1/10 vs. full data: ImageNette 47.7/63.0% (full 87.4%), ImageWoof 28.6/35.8% (full 67.0%), ImageFruit 26.6/40.3% (full 63.9%), ImageMeow 30.7/40.4% (full 66.7%), ImageSquawk 39.4/52.3% (full 87.5%), ImageYellow 45.2/60.0% (full 84.4%) -- the first trajectory-matching result above 64x64 resolution. Cross-architecture, CIFAR-10 IPC10, ConvNet-trained data evaluated on unseen nets: ResNet 46.4%, VGG 50.3%, AlexNet 34.2% (vs. DSA 42.8/43.2/35.9% and KIP 36.8/42.1/24.4%). Cost: expert trajectories took ~3 sec/epoch on CIFAR (8 GPU-hours total for 200 CIFAR experts, ~60MB/expert) and ~11 sec/epoch on the ImageNet subsets (15 GPU-hours for 100 experts, ~120MB/expert); the largest single distillation run used up to 144GB VRAM over 6xRTX6000 GPUs. Ablation (Table 5, CIFAR-100 IPC1): removing ZCA whitening, real-image initialization, or the learned synthetic step size each degrades accuracy below the full method's 24.3%, isolating the combination as necessary; the paper separately shows varying the matched expert-epoch offset $M$ and student steps $N$ both materially change accuracy, and that cached (precomputed) experts match live-training experts in accuracy at far lower per-run cost.", "limitations": "States explicitly that pre-computing and storing expert trajectories is a real up-front cost, both in compute (8-15 GPU-hours per dataset's expert pool) and disk (tens to hundreds of MB per expert), amortized across but not eliminated by reuse. Backpropagating through the $N$ synthetic-training steps is memory-intensive and scales with $N$ and with the number of images distilled jointly, requiring a per-class or per-batch chunking workaround (Eq. 3) and, in the largest configurations, up to 144GB of GPU memory (a bottleneck tesla-2023 directly targets next). Only hard labels and only ConvNet-family experts are used; no ImageNet-1K result (reached at 128x128 crops of 10-class subsets only); no soft-label or teacher-relabeling scheme." } }, { "id": "learning-to-generate-synthetic-2021", "type": "paper", "title": "Learning to Generate Synthetic Training Data using Gradient Matching and Implicit Differentiation", "arxiv": "2203.08559", "url": null, "date": "2022-03-16", "venue": { "name": "AIST 2021", "status": "conference" }, "authors": [ "Dmitry Medvedev", "Alexander D'yakonov" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "gradient-matching" ], "applications": [ "nas" ], "tier": "catalogued", "links": { "code": "https://github.com/dm-medvedev/efficientdistillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Combines ideas from Generative Teaching Networks, gradient matching and the Implicit Function Theorem into new data distillation techniques aimed at reducing training-data requirements. Reports the new methods are more computationally efficient than the prior techniques they combine and improve the performance of models trained on the distilled MNIST data.", "abstract": "Using huge training datasets can be costly and inconvenient. This article explores various data distillation techniques that can reduce the amount of data required to successfully train deep networks. Inspired by recent ideas, we suggest new data distillation techniques based on generative teaching networks, gradient matching, and the Implicit Function Theorem. Experiments with the MNIST image classification problem show that the new methods are computationally more efficient than previous ones and allow to increase the performance of models trained on distilled data.", "awesome_sections": [ "Applications / Neural Architecture Search" ], "notes": "" }, { "id": "cafe-2022", "type": "paper", "title": "CAFE: Learning to Condense Dataset by Aligning Features", "short": "CAFE", "arxiv": "2203.01531", "url": null, "date": "2022-03-03", "venue": { "name": "CVPR 2022", "status": "conference" }, "authors": [ "Kai Wang", "Bo Zhao", "Xiangyu Peng", "Zheng Zhu", "Shuo Yang", "Shuo Wang", "Guan Huang", "Hakan Bilen", "Xinchao Wang", "Yang You" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "tier": "core", "links": { "code": "https://github.com/kaiwang960112/cafe", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "parameterization": [ "pixels" ], "labels": "hard", "prior": [ "random-networks" ], "regime": "bilevel", "scale": [ "mnist-cifar" ] }, "explained": { "before": "Gradient matching (dc-2021) and its differentiable-augmentation refinement (dsa-2021) synthesize images by matching per-step gradients between real and synthetic batches inside a bi-level loop.", "problem": "The paper diagnoses that only a small number of hard or noisy samples produce dominant gradients late in training, so gradient matching overfits to those samples and lacks global supervision of the data distribution; because which samples are \"hard\" varies by architecture, this also hurts cross-architecture generalization.", "idea": "Explicitly align real and synthetic feature statistics at every layer of a network instead of matching gradients, keep a discrimination loss so the synthetic feature centers remain valid classifiers, and replace the fixed inner/outer loop counts of gradient matching with a dynamic schedule that stops each loop once validation accuracy stops improving.", "method": "Optimizes synthetic pixel images. A 3-layer ConvNet is alternately trained on the current synthetic set (inner loop, re-initialized periodically) and used to compute a layer-wise feature alignment loss -- MSE between real and synthetic class-wise mean feature maps at every convolutional layer -- plus a discrimination loss in which synthetic feature centers classify real samples via cross-entropy. A \"dynamic bi-level optimization\" (Algorithm 1) uses performance queues over recent validation accuracy to decide when to end the inner and outer loops, rather than a fixed iteration budget. The scheme is still bi-level: unlike the later distribution-matching methods it inspired, a network is trained on the synthetic data throughout synthesis.", "evidence": "SVHN, 1 IPC, ConvNet (avg of 100 trained networks): CAFE 42.6% vs. DC 31.2% (+11.4pp, Table 1). CIFAR-10, 50 IPC: CAFE 55.5% vs. DC 53.9%. CIFAR-100, 50 IPC: CAFE 37.9% vs. DC 42.8% (CAFE underperforms here). Training time (Table 5, CIFAR-10 IPC=50): DC ~460 min vs. CAFE ~367 min (gamma=10) or ~117 min (gamma=5) -- the speedup comes from adaptive early stopping of the loops, not from removing the inner loop. Ablation (Table 2, CIFAR-10 IPC=50): discrimination loss alone 49.78%, layer-wise feature alignment alone 53.96%, both 54.53%, adding the dynamic schedule 55.50%.", "limitations": "Only evaluated at MNIST/CIFAR/SVHN (32x32) scale, no ImageNet. Still requires training a network on the synthetic set throughout synthesis, so it does not reach the efficiency of the bi-level-free distribution matching (dm-2023) it is grouped with. Underperforms plain gradient matching (DC) on CIFAR-100 at 50 IPC. Cross-architecture accuracy remains modest (25-40%) despite the gain over DC. The dynamic schedule adds tunable thresholds (lambda_1, lambda_2, gamma) with a reported sensitivity study but no principled way to set them without a validation split." }, "builds_on": [ "dc-2021", "dsa-2021" ], "summary": "Aligns multi-scale feature statistics between real and synthetic batches instead of matching gradients directly, while preserving synthetic-set discriminability with a classification loss and a dynamic bi-level optimization schedule that adaptively prevents over- and under-fitting, reporting up to an 11-point accuracy gain over prior methods on SVHN.", "abstract": "Dataset condensation aims at reducing the network training effort through condensing a cumbersome training set into a compact synthetic one. State-of-the-art approaches largely rely on learning the synthetic data by matching the gradients between the real and synthetic data batches. Despite the intuitive motivation and promising results, such gradient-based methods, by nature, easily overfit to a biased set of samples that produce dominant gradients, and thus lack global supervision of data distribution. In this paper, we propose a novel scheme to Condense dataset by Aligning FEatures (CAFE), which explicitly attempts to preserve the real-feature distribution as well as the discriminant power of the resulting synthetic set, lending itself to strong generalization capability to various architectures. At the heart of our approach is an effective strategy to align features from the real and synthetic data across various scales, while accounting for the classification of real samples. Our scheme is further backed up by a novel dynamic bi-level optimization, which adaptively adjusts parameter updates to prevent over-/under-fitting. We validate the proposed CAFE across various datasets, and demonstrate that it generally outperforms the state of the art: on the SVHN dataset, for example, the performance gain is up to 11%. Extensive experiments and analyses verify the effectiveness and necessity of proposed designs.", "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ] }, { "id": "dcc-2022", "type": "paper", "title": "Dataset Condensation with Contrastive Signals", "short": "DCC", "arxiv": "2202.02916", "url": null, "date": "2022-02-07", "venue": { "name": "ICML 2022", "status": "conference" }, "authors": [ "Saehyung Lee", "Sanghyuk Chun", "Sangwon Jung", "Sangdoo Yun", "Sungroh Yoon" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "gradient-matching" ], "facets": { "parameterization": [ "pixels" ], "labels": "hard", "prior": [ "random-networks" ], "regime": "single-level", "scale": [ "mnist-cifar" ] }, "tier": "catalogued", "links": { "code": "https://github.com/saehyung-lee/dcc", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Shows that class-wise gradient matching can perform worse than random selection when the dataset contains a large share of task-irrelevant information, traces this to the loss function's lack of a contrastive signal between classes, and adds a modified loss plus a bi-level warm-up that recovers effective synthesis for fine-grained classification and improves over gradient-matching baselines on SVHN, CIFAR-10 and CIFAR-100.", "abstract": "Recent studies have demonstrated that gradient matching-based dataset synthesis, or dataset condensation (DC), methods can achieve state-of-the-art performance when applied to data-efficient learning tasks. However, in this study, we prove that the existing DC methods can perform worse than the random selection method when task-irrelevant information forms a significant part of the training dataset. We attribute this to the lack of participation of the contrastive signals between the classes resulting from the class-wise gradient matching strategy. To address this problem, we propose Dataset Condensation with Contrastive signals (DCC) by modifying the loss function to enable the DC methods to effectively capture the differences between classes. In addition, we analyze the new loss function in terms of training dynamics by tracking the kernel velocity. Furthermore, we introduce a bi-level warm-up strategy to stabilize the optimization. Our experimental results indicate that while the existing methods are ineffective for fine-grained image classification tasks, the proposed method can successfully generate informative synthetic datasets for the same tasks. Moreover, we demonstrate that the proposed method outperforms the baselines even on benchmark datasets such as SVHN, CIFAR-10, and CIFAR-100. Finally, we demonstrate the high applicability of the proposed method by applying it to continual learning tasks.", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ], "builds_on": [ "dc-2021" ], "explained": { "before": "Responds to dc-2021 (DC) and dsa-2021 (DSA), both of which match gradients class-by-class (each synthetic class's gradient is matched only to its own real class's gradient), and to selection-based baselines (random, C-score, GraNd, EL2N) and kip-2021 (KIP), which it also compares against directly on fine-grained subsets of ImageNet classes.", "problem": "Proves formally and empirically that class-wise gradient matching can perform *worse* than random real-image selection when a class's images are dominated by task-irrelevant, class-common features (e.g. wheels, headlights and roads shared by every \"truck\" image) rather than task-relevant, class-discriminative features (a truck's logo or trailer shape). Because the matching loss for each class is computed independently, nothing in the objective tells it to prefer the discriminative features over the common ones, so it can spend the synthetic set's limited capacity on information that does not help classification.", "idea": "Add an inter-class contrastive term to the gradient-matching loss so that, alongside matching each class's gradient to its own class's real gradient, the loss also explicitly discourages the synthetic set from producing gradients that other classes could equally well produce -- steering the limited synthetic capacity toward class-discriminative rather than class-common information. Stabilize the resulting harder optimization with a short bi-level warm-up phase before switching to the single-level contrastive matching.", "method": "Builds on dc-2021's per-step gradient-matching objective (network re-initialized repeatedly; gradients on synthetic vs. real class batches matched via a distance $D$), but replaces the purely class-wise loss with a modified objective (Sec. 3) that adds a contrastive term pushing a class's synthetic gradient away from other classes' real gradients, not just toward its own. Training begins with a short bi-level warm-up phase (directly optimizing a short unrolled objective, dd-2018-style) before switching to the single-level contrastive matching loss for the remainder of synthesis, which the paper shows via tracked kernel-velocity analysis stabilizes early optimization. DCC denotes the method applied to plain (unaugmented) matching; DSAC denotes the same contrastive loss combined with dsa-2021's Siamese differentiable augmentation. Only per-class synthetic images (pixels, hard labels) are stored; no extra artifact beyond dc-2021's.", "evidence": "Table 2 (fine-grained ImageNet-derived classes, ConvNet, hard labels; ipc=10 shown): on Automobile, Random 12.2% vs. DC 11.0% (DC *below* random) vs. DSA 19.1% vs. DCC 18.6% vs. DSAC 22.1%; on Terrier, Random 5.6% vs. DC 4.6% (below random) vs. DSA 5.1% vs. DCC 6.4% vs. DSAC 6.2%; on Fish, Random 14.7% vs. DC 13.5% (below random) vs. DSA 18.7% vs. DCC 20.4% vs. DSAC 22.3% -- DC falls below random selection on 3 of 6 fine-grained subsets tested, while DCC/DSAC recover above random and above DC/DSA in every case. Table 3 (SVHN/CIFAR-10/CIFAR-100, DC/DSA/DCC/DSAC): SVHN IPC1/10/50 -- DC 34.6/76.2/ 82.7%, DSA 36.0/78.9/84.4%, DCC 34.3/76.2/83.3%, DSAC 47.5/80.5/87.2% (full 92.1+/-0.2%); CIFAR-10 IPC1/10/50 -- DC 28.2/44.7/54.8%, DSA 28.7/52.1/60.6%, DCC 32.9/49.4/61.6%, DSAC 34.0/54.5/64.2% (full 81.6+/-0.3%); CIFAR-100 IPC1/10/50 -- DC 12.8/26.6/32.1%, DSA 13.9/32.4/38.6%, DCC 13.3/30.6/ 40.0%, DSAC 14.6/33.5/39.3% (full 52.5+/-0.3%) -- DCC beats DC at every setting; DSAC beats DSA at every setting except CIFAR-100 IPC50 (39.3 vs. 38.6, a narrow win) and CIFAR-10 IPC10 where DCC alone trails DSA (49.4 vs. 52.1, showing the contrastive term alone is not always enough without augmentation). Cross-architecture (condensed data evaluated across ConvNet/LeNet/AlexNet/VGG/ResNet): DSAC reaches 64.1/42.6/48.2/56.0/53.9% vs. DSA's 60.4/40.3/46.0/50.7/49.7% and DC's 54.8/33.8/40.9/ 39.3/23.9% -- consistently ahead at every architecture. Ablation (Table, bi-level warm-up on/off, CIFAR-10/100 IPC1/10/50): warm-up adds 0.1-4.6 points depending on setting (e.g. CIFAR-10 IPC1 DCC 28.3->32.9 with warm-up; CIFAR-100 IPC10 DSAC 29.3->33.5), confirming the warm-up's stabilizing effect though the gain is uneven across settings and datasets.", "limitations": "The below-random-selection critique and the contrastive fix are demonstrated mainly on constructed fine-grained/task-irrelevant-heavy ImageNet subsets and standard SVHN/CIFAR-10/100 benchmarks, not at Tiny-ImageNet or larger scale; still single-step gradient matching at its core, inheriting the short-horizon-signal weakness the trajectory-matching family was built to address. The contrastive term alone (DCC) does not always beat augmentation alone (DSA) -- e.g. CIFAR-10 IPC10, DCC trails DSA -- so the strongest results (DSAC) require combining both fixes rather than the contrastive signal replacing augmentation. No GPU-hour or memory figure reported." } }, { "id": "wearable-imagenet-synthesizing-tileable-2022", "type": "paper", "title": "Wearable ImageNet: Synthesizing Tileable Textures via Dataset Distillation", "arxiv": null, "url": "https://openaccess.thecvf.com/content/CVPR2022W/CVFAD/html/Cazenavette_Wearable_ImageNet_Synthesizing_Tileable_Textures_via_Dataset_Distillation_CVPRW_2022_paper.html", "date": "2022-01-01", "venue": { "name": "CVPR 2022 Workshop", "status": "workshop" }, "authors": [ "George Cazenavette", "Tongzhou Wang", "Antonio Torralba", "Alexei A. Efros", "Jun-Yan Zhu" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "trajectory-matching" ], "applications": [ "fashion" ], "tier": "catalogued", "links": { "code": "https://github.com/georgecazenavette/mtt-distillation", "project": "https://georgecazenavette.github.io/mtt-distillation/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "cvf", "depth": "abstract" }, "summary": "Generates tileable distilled \"textures\" by sampling random crops from a toroidal canvas of trajectory-matching-distilled pixels while enforcing that every crop is itself effective distilled training data for its class, producing infinite repeating patterns that visually summarize an ImageNet category and are suitable for printing on fabric or clothing.", "abstract": null, "awesome_sections": [ "Applications / Fashion, Art, and Design" ], "notes": "" }, { "id": "learning-from-designers-fashion-2022", "type": "paper", "title": "Learning from Designers: Fashion Compatibility Analysis Via Dataset Distillation", "short": "ADD-GNN", "arxiv": null, "url": "https://ieeexplore.ieee.org/document/9897234/", "date": "2022-01-01", "venue": { "name": "ICIP 2022", "status": "conference" }, "authors": [ "Yulan Chen" ], "kind": "application", "settings": [ "image-classification" ], "families": [], "applications": [ "fashion" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "ieee", "depth": "abstract" }, "summary": "Proposes an Attention-based Dataset Distillation Graph Neural Network (ADD-GNN) that distills knowledge from designer-generated outfit data, rather than noisy user-generated social-media data, to guide feature representation learning, and models overall outfit compatibility with an attention-based graph neural network jointly optimized with the distillation component.", "abstract": null, "awesome_sections": [ "Applications / Fashion, Art, and Design" ], "notes": "Full author list not confirmed beyond the first author; IEEE Xplore abstract page returned only a partial description via search, not the full text (families left empty: the \"distillation\" here is a designer-knowledge-transfer mechanism specific to this paper, not one of the atlas condensation families)." }, { "id": "dense-2022", "type": "paper", "title": "DENSE: Data-Free One-Shot Federated Learning", "short": "DENSE", "arxiv": "2112.12371", "url": null, "date": "2021-12-23", "venue": { "name": "NeurIPS 2022", "status": "conference" }, "authors": [ "Jie Zhang", "Chen Chen", "Bo Li", "Lingjuan Lyu", "Shuang Wu", "Shouhong Ding", "Chunhua Shen", "Chao Wu" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "decoupled-synthesis" ], "facets": { "prior": [ "pretrained-classifier" ], "regime": "decoupled" }, "applications": [ "federated-learning" ], "tier": "notable", "links": { "code": "https://github.com/zj-jayzhang/DENSE", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "For one-shot federated learning with heterogeneous client architectures, DENSE trains a generator to invert an ensemble of the uploaded client models into synthetic images, then distills that ensemble's knowledge into a single global model on the synthetic data, needing no public dataset and no extra information beyond model parameters. Reports outperforming the strongest prior one-shot FL baseline by 5.08% on CIFAR-10.", "abstract": "One-shot Federated Learning (FL) has recently emerged as a promising approach, which allows the central server to learn a model in a single communication round. Despite the low communication cost, existing one-shot FL methods are mostly impractical or face inherent limitations, \\eg a public dataset is required, clients' models are homogeneous, and additional data/model information need to be uploaded. To overcome these issues, we propose a novel two-stage \\textbf{D}ata-fre\\textbf{E} o\\textbf{N}e-\\textbf{S}hot federated l\\textbf{E}arning (DENSE) framework, which trains the global model by a data generation stage and a model distillation stage. DENSE is a practical one-shot FL method that can be applied in reality due to the following advantages: (1) DENSE requires no additional information compared with other methods (except the model parameters) to be transferred between clients and the server; (2) DENSE does not require any auxiliary dataset for training; (3) DENSE considers model heterogeneity in FL, \\ie different clients can have different model architectures. Experiments on a variety of real-world datasets demonstrate the superiority of our method.For example, DENSE outperforms the best baseline method Fed-ADI by 5.08\\% on CIFAR10 dataset.", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "" }, { "id": "gcond-2022", "type": "paper", "title": "Graph Condensation for Graph Neural Networks", "short": "GCond", "arxiv": "2110.07580", "url": null, "date": "2021-10-14", "venue": { "name": "ICLR 2022", "status": "conference" }, "authors": [ "Wei Jin", "Lingxiao Zhao", "Shichang Zhang", "Yozen Liu", "Jiliang Tang", "Neil Shah" ], "kind": "method", "settings": [ "graph" ], "families": [ "gradient-matching" ], "facets": { "parameterization": [ "graph-structure" ], "labels": "hard", "prior": [ "random-networks" ], "regime": "single-level", "scale": [ "large-scale-other" ] }, "builds_on": [ "dc-2021" ], "tier": "landmark", "links": { "code": "https://github.com/chandlerbang/gcond", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Introduces GCond, which condenses a large graph into a small synthetic graph and features by imitating the GNN training trajectory on the original graph through a gradient-matching loss, jointly synthesizing node features and structure so that GNNs trained on the condensed graph transfer across architectures; reports approximating original test accuracy by 95.3% on Reddit, 99.8% on Flickr and 99.0% on Citeseer while shrinking graph size by over 99.9%.", "abstract": "Given the prevalence of large-scale graphs in real-world applications, the storage and time for training neural models have raised increasing concerns. To alleviate the concerns, we propose and study the problem of graph condensation for graph neural networks (GNNs). Specifically, we aim to condense the large, original graph into a small, synthetic and highly-informative graph, such that GNNs trained on the small graph and large graph have comparable performance. We approach the condensation problem by imitating the GNN training trajectory on the original graph through the optimization of a gradient matching loss and design a strategy to condense node futures and structural information simultaneously. Extensive experiments have demonstrated the effectiveness of the proposed framework in condensing different graph datasets into informative smaller graphs. In particular, we are able to approximate the original test accuracy by 95.3% on Reddit, 99.8% on Flickr and 99.0% on Citeseer, while reducing their graph size by more than 99.9%, and the condensed graphs can be used to train various GNN architectures.Code is released at https://github.com/ChandlerBang/GCond.", "awesome_sections": [ "Applications / Graph Neural Network" ], "notes": "", "explained": { "before": "Prior graph-size reduction was graph sparsification (drop edges to preserve pairwise distances/cuts/eigenvalues) and graph coarsening (group nodes into super-nodes), neither of which is optimized for downstream GNN accuracy and neither of which reduces node attributes. Dataset distillation (DD) and dataset condensation (DC, gradient matching) had solved the analogous problem for images but assume independent, Euclidean samples.", "problem": "Image-style condensation cannot be applied to graphs because nodes are interdependent through edges (message passing couples every node to its neighborhood), so there is no notion of condensing \"one sample\" independently of the rest of the graph; the paper also observes that the discrete adjacency matrix has O(N'^2) parameters if learned freely, which overfits as the condensed graph grows.", "idea": "Treat graph condensation as the same gradient-matching bi-level problem as image DC (match per-step GNN parameter gradients between condensed and real data across random initializations), but resolve the graph-specific coupling by parameterizing the condensed adjacency matrix as a function of the condensed node features (an MLP over pairs of feature vectors) instead of a free parameter, so structure and features are optimized jointly but structure never needs its own O(N'^2) parameter block.", "method": "Optimizes condensed features $\\mathbf{X}'$ and an MLP $g_\\Phi$ that outputs $\\mathbf{A}'=g_\\Phi(\\mathbf{X}')$ (symmetrized sigmoid of an MLP over concatenated node-feature pairs); condensed labels $\\mathbf{Y}'$ are fixed to match the real class distribution and never learned. In the loop: a GNN (SGC used for the condensation objective, GCN for evaluation) is initialized from a distribution $P_{\\theta_0}$, trained for one step at a time on both the condensed graph and a class-wise sampled batch of real nodes (with sampled neighbors), and $\\mathbf{X}'$/$\\Phi$ are updated to minimize the cosine-distance gradient-matching loss $\\sum_t D(\\nabla_\\theta \\mathcal{L}(\\mathrm{GNN}_{\\theta_t}(\\mathbf{A}',\\mathbf{X}'),\\mathbf{Y}'), \\nabla_\\theta \\mathcal{L}(\\mathrm{GNN}_{\\theta_t}(\\mathbf{A},\\mathbf{X}),\\mathbf{Y}))$, summed per class to ease optimization. $\\mathbf{X}'$ and $\\Phi$ are updated in alternating blocks of epochs rather than jointly. After training, small edge weights in $\\mathbf{A}'$ are thresholded away to sparsify storage. A \"graphless\" variant, GCond-x, fixes $\\mathbf{A}'=\\mathbf{I}$ and only learns $\\mathbf{X}'$.", "evidence": "Table 2 (GCN evaluator, hard labels, node classification, 10 repeats): on Cora at r=1.3% GCond reaches 79.8% vs. 81.2% whole-dataset accuracy; on Reddit at r=0.1% GCond reaches 89.6% vs. 93.9% whole (the abstract's headline 95.3%-of-original number); on Flickr at r=1% GCond reaches 47.1% vs. 47.1% whole (matches exactly); the largest gap is ogbn-arxiv at r=0.05%, 61.3% vs. 71.4% whole. GCond beats random, herding, k-center and graph-coarsening baselines by a large margin at the smallest ratios (Obs 1). Table 3/4 show cross-architecture transfer (APPNP/Cheby/GCN/GraphSAGE/SGC/MLP/GAT) with no more than a few points of drop. Table 5 reports storage: Reddit condenses from 435.5MB/153,932 nodes to 0.4MB/153 nodes. Key ablation: GCond-x (features only, identity adjacency) matches GCond within a point or two on 11 of 15 dataset/ratio combinations, showing that most of the gradient-matching signal is absorbed into node features even when no explicit structure is learned; explicit structure mainly helps cross-architecture generalization and interpretability rather than raw accuracy.", "limitations": "The gap to whole-dataset accuracy grows on the largest, most heterophilous graph tested (ogbn-arxiv); node labels are fixed rather than learned/relabeled; only single-label transductive/inductive node classification is addressed, with graph-level classification and multi-label settings left as future work (stated in the conclusion); jointly optimizing features and structure is unstable enough to require an alternating-block schedule rather than joint gradient descent; and the learned structure is sometimes uninterpretable (a near star-graph on Reddit), suggesting the structure channel can degenerate to a trivial solution the accuracy metric cannot detect." } }, { "id": "dm-2023", "type": "paper", "title": "Dataset Condensation with Distribution Matching", "short": "DM", "arxiv": "2110.04181", "url": null, "date": "2021-10-08", "venue": { "name": "WACV 2023", "status": "conference" }, "authors": [ "Bo Zhao", "Hakan Bilen" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "distribution-matching" ], "tier": "landmark", "links": { "code": "https://github.com/VICO-UoE/DatasetCondensation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "facets": { "parameterization": [ "pixels" ], "labels": "hard", "prior": [ "random-networks" ], "regime": "single-level", "scale": [ "mnist-cifar", "tiny-imagenet", "imagenet-1k" ] }, "builds_on": [ "dd-2018", "dc-2021", "dsa-2021" ], "explained": { "before": "Meta-learning dataset distillation (dd-2018) and gradient matching (dc-2021, dsa-2021) both require expensive bi-level optimization; the paper reports DSA needs about 15 GPU-hours to learn 500 CIFAR-10 images -- comparable to training six deep networks -- and needs several hyperparameters retuned per setting.", "problem": "Bi-level optimization, even without full unrolling, is too costly to scale condensation to larger, more realistic datasets and architectures.", "idea": "If the synthetic set's per-class feature distribution matches the real data's in the embedding space of many different (mostly randomly initialized) networks, a model trained on it should behave like one trained on the real data -- and this can be enforced directly with no inner-loop training at all.", "method": "Optimizes synthetic pixel images (initialized from randomly sampled real images) by minimizing, for each of many randomly sampled network architectures/initializations, the squared distance between real and synthetic class-wise mean embeddings (an empirical MMD with a linear kernel) computed after Differentiable Siamese Augmentation. No network is ever trained; only the synthetic images receive gradients. Effectively a single hyperparameter (the image learning rate) needs tuning across settings.", "evidence": "CIFAR-10, 50 IPC, ConvNet: DM 63.0% vs. DSA 60.6% (Table 1). CIFAR-100, 50 IPC: 43.6% vs. DSA 42.8%. TinyImageNet, 50 IPC: 24.1% vs. Herding 16.7%. ImageNet-1K, 50 IPC: 11.4% (vs. random selection 7.54%). Cross-architecture (CIFAR-10, 50 IPC, trained on ConvNet, evaluated on AlexNet/VGG/ResNet, Table 3): 61.3%/59.9%/57.0% vs. DSA 53.3%/51.0%/47.3%. Continual learning (CIFAR-100, 5-step): 34.4% vs. DSA 31.7%. Cost: under 20 minutes for CIFAR-10 IPC=50 vs. DSA's 15 hours (over 45x speedup, \"under 2.2% of the training cost\"); TinyImageNet took 27 hours on one V100, ImageNet-1K 28 hours on ten GTX 1080s. Ablation (Table 4): nine different distributions of sampling networks (from purely random to well-trained) all give similar accuracy on CIFAR-10, showing the objective is not sensitive to which networks are sampled. Batch norm instead of instance norm gains +2.2pp (CIFAR-10) and +4.4pp (CIFAR-100).", "limitations": "At 1 image/class, DSA outperforms DM on some datasets. Only the first moment (class mean) of the feature distribution is matched; higher moments are left free, which idm-2023, m3d-2024 and datadam-2023 later target directly. Because synthetic images are initialized from and stay close to real images, gains diminish at large IPC as the initial distribution gap shrinks. More sophisticated evaluation architectures (ResNet) yield worse condensation quality than simple ConvNets, indicating some architecture-specific bias." }, "summary": "Introduces distribution matching: instead of matching gradients or unrolling training, synthesize images so their class-wise feature means match those of real images across many randomly sampled embedding networks, removing the bi-level optimization entirely and making condensation cheap enough to scale to larger, more realistic datasets.", "abstract": "Computational cost of training state-of-the-art deep models in many learning problems is rapidly increasing due to more sophisticated models and larger datasets. A recent promising direction for reducing training cost is dataset condensation that aims to replace the original large training set with a significantly smaller learned synthetic set while preserving the original information. While training deep models on the small set of condensed images can be extremely fast, their synthesis remains computationally expensive due to the complex bi-level optimization and second-order derivative computation. In this work, we propose a simple yet effective method that synthesizes condensed images by matching feature distributions of the synthetic and original training images in many sampled embedding spaces. Our method significantly reduces the synthesis cost while achieving comparable or better performance. Thanks to its efficiency, we apply our method to more realistic and larger datasets with sophisticated neural architectures and obtain a significant performance boost. We also show promising practical benefits of our method in continual learning and neural architecture search.", "awesome_sections": [ "Main / Distribution/Feature Matching Surrogate Objective" ] }, { "id": "kip-conv-2021", "type": "paper", "title": "Dataset Distillation with Infinitely Wide Convolutional Networks", "short": "KIP-ConvNet", "arxiv": "2107.13034", "url": null, "date": "2021-07-27", "venue": { "name": "NeurIPS 2021", "status": "conference" }, "authors": [ "Timothy Nguyen", "Roman Novak", "Lechao Xiao", "Jaehoon Lee" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "kernel-closed-form" ], "tier": "notable", "links": { "code": "https://github.com/google/neural-tangents", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Extends KIP's kernel-ridge-regression meta-learning to infinitely wide convolutional networks via a distributed NTK computation, reporting over 65% CIFAR-10 test accuracy from just 10 datapoints — up from a prior best of 40% — with matching state-of-the-art results across MNIST, Fashion-MNIST, CIFAR-10, CIFAR-100 and SVHN.", "abstract": "The effectiveness of machine learning algorithms arises from being able to extract useful features from large amounts of data. As model and dataset sizes increase, dataset distillation methods that compress large datasets into significantly smaller yet highly performant ones will become valuable in terms of training efficiency and useful feature extraction. To that end, we apply a novel distributed kernel based meta-learning framework to achieve state-of-the-art results for dataset distillation using infinitely wide convolutional neural networks. For instance, using only 10 datapoints (0.02% of original dataset), we obtain over 65% test accuracy on CIFAR-10 image classification task, a dramatic improvement over the previous best test accuracy of 40%. Our state-of-the-art results extend across many other settings for MNIST, Fashion-MNIST, CIFAR-10, CIFAR-100, and SVHN. Furthermore, we perform some preliminary analyses of our distilled datasets to shed light on how they differ from naturally occurring data.", "awesome_sections": [ "Main / Kernel-Based Distillation" ], "builds_on": [ "kip-2021" ] }, { "id": "data-diet-2021", "type": "paper", "title": "Deep Learning on a Data Diet: Finding Important Examples Early in Training", "short": "EL2N / GraNd", "arxiv": "2107.07075", "url": null, "date": "2021-07-15", "venue": { "name": "NeurIPS 2021", "status": "conference" }, "authors": [ "Mansheej Paul", "Surya Ganguli", "Gintare Karolina Dziugaite" ], "kind": "baseline", "settings": [ "image-classification" ], "families": [ "coreset-selection" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Introduces two per-example difficulty scores computable early in training, the Gradient Norm (GraNd) and the Error L2-Norm (EL2N), and shows they identify which training examples can be pruned without hurting generalization. Reports pruning half of CIFAR-10 with EL2N scores from a few epochs of training while slightly improving test accuracy, and that scores computed with one architecture generalize to others.", "abstract": "Recent success in deep learning has partially been driven by training increasingly overparametrized networks on ever larger datasets. It is therefore natural to ask: how much of the data is superfluous, which examples are important for generalization, and how do we find them? In this work, we make the striking observation that, in standard vision datasets, simple scores averaged over several weight initializations can be used to identify important examples very early in training. We propose two such scores -- the Gradient Normed (GraNd) and the Error L2-Norm (EL2N) scores -- and demonstrate their efficacy on a range of architectures and datasets by pruning significant fractions of training data without sacrificing test accuracy. In fact, using EL2N scores calculated a few epochs into training, we can prune half of the CIFAR10 training set while slightly improving test accuracy. Furthermore, for a given dataset, EL2N scores from one architecture or hyperparameter configuration generalize to other configurations. Compared to recent work that prunes data by discarding examples that are rarely forgotten over the course of training, our scores use only local information early in training. We also use our scores to detect noisy examples and study training dynamics through the lens of important examples -- we investigate how the data distribution shapes the loss surface and identify subspaces of the model's data representation that are relatively stable over training.", "notes": "" }, { "id": "data-distillation-for-text-2021", "type": "paper", "title": "Data Distillation for Text Classification", "arxiv": "2104.08448", "url": null, "date": "2021-04-17", "venue": { "name": "arXiv 2021", "status": "preprint" }, "authors": [ "Yongqi Li", "Wenjie Li" ], "kind": "method", "settings": [ "text" ], "families": [], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Proposes an early data-distillation method for text classification that learns a small set of synthetic training examples; reports on eight benchmark datasets that a distilled set 0.1% the size of the original text data recovers about 90% of full-data performance.", "abstract": "Deep learning techniques have achieved great success in many fields, while at the same time deep learning models are getting more complex and expensive to compute. It severely hinders the wide applications of these models. In order to alleviate this problem, model distillation emerges as an effective means to compress a large model into a smaller one without a significant drop in accuracy. In this paper, we study a related but orthogonal issue, data distillation, which aims to distill the knowledge from a large training dataset down to a smaller and synthetic one. It has the potential to address the large and growing neural network training problem based on the small dataset. We develop a novel data distillation method for text classification. We evaluate our method on eight benchmark datasets. The results that the distilled data with the size of 0.1% of the original text data achieves approximately 90% performance of the original is rather impressive.", "awesome_sections": [ "Applications / Text" ], "notes": "Abstract does not name a specific matching mechanism; family left empty." }, { "id": "gastric-sldd-2020", "type": "paper", "title": "Soft-Label Anonymous Gastric X-ray Image Distillation", "short": "Gastric SLDD", "arxiv": "2104.02857", "url": null, "date": "2021-04-07", "venue": { "name": "ICIP 2020", "status": "conference" }, "authors": [ "Guang Li", "Ren Togo", "Takahiro Ogawa", "Miki Haseyama" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "meta-learning-bptt" ], "facets": { "labels": "learned" }, "applications": [ "medical", "privacy" ], "tier": "catalogued", "links": { "code": "https://github.com/Guang000/dataset-distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Applies gradient-descent dataset distillation, jointly optimizing distilled images, distilled soft labels and the learning rate, to gastric X-ray images so the resulting tiny distilled set both compresses the dataset and anonymizes the original patient images by construction. Reports the compressed images no longer carry identifiable patient information while remaining useful for training.", "abstract": "This paper presents a soft-label anonymous gastric X-ray image distillation method based on a gradient descent approach. The sharing of medical data is demanded to construct high-accuracy computer-aided diagnosis (CAD) systems. However, the large size of the medical dataset and privacy protection are remaining problems in medical data sharing, which hindered the research of CAD systems. The idea of our distillation method is to extract the valid information of the medical dataset and generate a tiny distilled dataset that has a different data distribution. Different from model distillation, our method aims to find the optimal distilled images, distilled labels and the optimized learning rate. Experimental results show that the proposed method can not only effectively compress the medical dataset but also anonymize medical images to protect the patient's private information. The proposed approach can improve the efficiency and security of medical data sharing.", "awesome_sections": [ "Applications / Medical" ], "notes": "" }, { "id": "distilled-replay-overcoming-forgetting-2021", "type": "paper", "title": "Distilled Replay: Overcoming Forgetting through Synthetic Samples", "arxiv": "2103.15851", "url": null, "date": "2021-03-29", "venue": { "name": "IJCAI 2021 Workshop", "status": "workshop" }, "authors": [ "Andrea Rosasco", "Antonio Carta", "Andrea Cossu", "Vincenzo Lomonaco", "Davide Bacciu" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "meta-learning-bptt" ], "applications": [ "continual-learning" ], "tier": "catalogued", "links": { "code": "https://github.com/andrearosasco/DistilledReplay", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Builds a rehearsal buffer for continual learning by distilling the full dataset into one highly informative synthetic pattern per class, rather than storing real exemplars, keeping the buffer extremely small. Reports effectiveness against popular replay-based continual learning baselines on four benchmarks.", "abstract": "Replay strategies are Continual Learning techniques which mitigate catastrophic forgetting by keeping a buffer of patterns from previous experiences, which are interleaved with new data during training. The amount of patterns stored in the buffer is a critical parameter which largely influences the final performance and the memory footprint of the approach. This work introduces Distilled Replay, a novel replay strategy for Continual Learning which is able to mitigate forgetting by keeping a very small buffer (1 pattern per class) of highly informative samples. Distilled Replay builds the buffer through a distillation process which compresses a large dataset into a tiny set of informative examples. We show the effectiveness of our Distilled Replay against popular replay-based strategies on four Continual Learning benchmarks.", "awesome_sections": [ "Applications / Continual Learning" ], "notes": "" }, { "id": "condensed-composite-memory-continual-2021", "type": "paper", "title": "Condensed Composite Memory Continual Learning", "short": "CCMCL", "arxiv": "2102.09890", "url": null, "date": "2021-02-19", "venue": { "name": "IJCNN 2021", "status": "conference" }, "authors": [ "Felix Wiewel", "Bin Yang" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "meta-learning-bptt", "parameterization" ], "facets": { "parameterization": [ "factorized-bases" ] }, "applications": [ "continual-learning" ], "tier": "catalogued", "links": { "code": "https://github.com/FelixWiewel/CCMCL", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "For rehearsal-based continual learning, learns a small set of shared components combined into per-class synthetic exemplars rather than learning each replay image directly, increasing memory efficiency of the stored rehearsal buffer. Reports improved performance over prior rehearsal baselines, especially at very small buffer sizes.", "abstract": "Deep Neural Networks (DNNs) suffer from a rapid decrease in performance when trained on a sequence of tasks where only data of the most recent task is available. This phenomenon, known as catastrophic forgetting, prevents DNNs from accumulating knowledge over time. Overcoming catastrophic forgetting and enabling continual learning is of great interest since it would enable the application of DNNs in settings where unrestricted access to all the training data at any time is not always possible, e.g. due to storage limitations or legal issues. While many recently proposed methods for continual learning use some training examples for rehearsal, their performance strongly depends on the number of stored examples. In order to improve performance of rehearsal for continual learning, especially for a small number of stored examples, we propose a novel way of learning a small set of synthetic examples which capture the essence of a complete dataset. Instead of directly learning these synthetic examples, we learn a weighted combination of shared components for each example that enables a significant increase in memory efficiency. We demonstrate the performance of our method on commonly used datasets and compare it to recently proposed related methods and baselines.", "awesome_sections": [ "Applications / Continual Learning" ], "notes": "" }, { "id": "dsa-2021", "type": "paper", "title": "Dataset Condensation with Differentiable Siamese Augmentation", "short": "DSA", "arxiv": "2102.08259", "url": null, "date": "2021-02-16", "venue": { "name": "ICML 2021", "status": "conference" }, "authors": [ "Bo Zhao", "Hakan Bilen" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "gradient-matching" ], "facets": { "parameterization": [ "pixels" ], "labels": "hard", "prior": [ "random-networks" ], "regime": "single-level", "scale": [ "mnist-cifar" ] }, "tier": "landmark", "links": { "code": "https://github.com/VICO-UoE/DatasetCondensation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Adds differentiable Siamese augmentation — applying the same random augmentation to a real and a synthetic batch before computing the gradient-matching loss — reporting roughly 7-point accuracy gains on CIFAR-10 and CIFAR-100 over the unaugmented gradient-matching baseline.", "abstract": "In many machine learning problems, large-scale datasets have become the de-facto standard to train state-of-the-art deep networks at the price of heavy computation load. In this paper, we focus on condensing large training sets into significantly smaller synthetic sets which can be used to train deep neural networks from scratch with minimum drop in performance. Inspired from the recent training set synthesis methods, we propose Differentiable Siamese Augmentation that enables effective use of data augmentation to synthesize more informative synthetic images and thus achieves better performance when training networks with augmentations. Experiments on multiple image classification benchmarks demonstrate that the proposed method obtains substantial gains over the state-of-the-art, 7% improvements on CIFAR10 and CIFAR100 datasets. We show with only less than 1% data that our method achieves 99.6%, 94.9%, 88.5%, 71.5% relative performance on MNIST, FashionMNIST, SVHN, CIFAR10 respectively. We also explore the use of our method in continual learning and neural architecture search, and show promising results.", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ], "builds_on": [ "dc-2021" ], "explained": { "before": "Responds directly to dc-2021 (DC), whose gradient matching is done on raw, unaugmented images even though the resulting condensed set is later trained with standard data augmentation, and to dd-2018 (DD) and LD (label distillation), which DSA also compares against on the same benchmarks.", "problem": "Standard data augmentation (crop, flip, color, etc.) usually helps training, but applying it naively to a condensed set is wasted or even harmful: the images were optimized without ever \"seeing\" augmentation, so an augmented view of a synthetic image may no longer carry the same information the gradient-matching objective packed into the original pixels.", "idea": "Make the condensed images \"augmentation-ready\" by exposing gradient matching itself to augmentation: at every matching step, sample one random augmentation and apply the *same* transformation (parameterized by the same random draw $\\omega$) to both the real and the synthetic batch before computing their gradients, so the images are optimized to remain informative under the transformation rather than only in their un-augmented form.", "method": "Keeps dc-2021's per-class synthetic image set (pixels, hard labels) and per-step gradient-matching loss $D(\\nabla_\\theta\\mathcal{L}(\\mathcal{S},\\theta_t),\\nabla_\\theta\\mathcal{L}(\\mathcal{T},\\theta_t))$, but replaces the loss's inputs with gradients computed on augmented batches $\\mathcal{A}(\\mathcal{S},\\omega)$ and $\\mathcal{A}(\\mathcal{T},\\omega)$, where $\\omega$ (the augmentation's random parameters -- crop location, rotation angle, etc.) is drawn once per step and applied identically (\"Siamese\") to both batches (Eq. 4-5, summed over a family of augmentation types $\\Omega$). Everything else -- repeated random network initializations, one forward-backward pass per step, no unrolling -- is unchanged from dc-2021.", "evidence": "Table 1 (ConvNet, hard labels): MNIST IPC1/10/50 = 88.7+/-0.6 / 97.8+/-0.1 / 99.2+/-0.1% (full 99.6+/-0.0%, DC 91.7/97.4/98.8%); FashionMNIST IPC1/10/50 = 70.6+/-0.6 / 84.6+/-0.3 / 88.7+/-0.2% (full 93.5+/-0.1%, DC 70.5/82.3/83.6%); SVHN IPC1/10/50 = 27.5+/-1.4 / 79.2+/-0.5 / 84.4+/-0.4% (full 95.4+/-0.1%, DC 31.2/76.1/82.3% -- DSA is slightly below DC at IPC1 here); CIFAR-10 IPC1/10/50 = 28.8+/-0.7 / 52.1+/-0.5 / 60.6+/-0.5% (full 84.8+/-0.1%, DC 28.3/44.9/53.9% -- the roughly 7-10 point CIFAR-10 gain over DC at IPC10/50 that the paper's abstract headlines). CIFAR-100 IPC1/10 = 13.9+/-0.3 / 32.3+/-0.3% (full 56.2+/-0.3%, DC 12.8/25.2%). States \"less than 1% data\" (50/class) reaches 99.6, 94.9, 88.5, 71.5% *relative* to full-data accuracy on MNIST/FashionMNIST/SVHN/CIFAR-10 respectively. Cross-architecture (Table 3, condensed on row architecture C, evaluated on column architecture T, CIFAR-10 IPC10): condensing on ConvNet transfers at 75.6-89.3% across MLP/ConvNet/LeNet/AlexNet/ VGG/ResNet, with all six condensation architectures transferring within a similar band -- the paper's evidence that DSA's gain is not architecture-specific. Ablation (Table 5, CIFAR-10 IPC50, evaluated under four different test-time augmentations): the full method (Siamese augmentation at condense time, standard augmentation at test time) reaches 49.1/47.9/46.9/46.8% under Crop/Flip/Scale/Rotation vs. 45.5% flat with no augmentation anywhere (variant A); applying augmentation independently (not Siamese) to the real and synthetic batches during condensing (variant C, 42.8/46.2/44.5/44.5%) or only at test time with no condense-time augmentation (variant B, 46.9/46.1/45.7/45.0%) both underperform the shared/Siamese variant -- isolating that *consistency* between the real and synthetic augmentation during matching, not just more augmented data, drives the gain.", "limitations": "Still inherits dc-2021's single-step, short-horizon gradient-matching objective and its dependence on the specific matching architecture; only tested at the same MNIST/FashionMNIST/SVHN/CIFAR-10/100 small-scale benchmarks as dc-2021, no Tiny-ImageNet or ImageNet result. The augmentation family $\\Omega$ (crop, flip, scale, rotation, color, cutout) and its parameterization are fixed choices, not learned; DSA slightly underperforms DC on SVHN at IPC1 (27.5% vs. 31.2%), showing the augmentation fix is not a uniform win at the smallest budgets. No GPU-hour or memory figure is reported." } }, { "id": "kip-2021", "type": "paper", "title": "Dataset Meta-Learning from Kernel Ridge-Regression", "short": "KIP", "arxiv": "2011.00050", "url": null, "date": "2020-10-30", "venue": { "name": "ICLR 2021", "status": "conference" }, "authors": [ "Timothy Nguyen", "Zhourong Chen", "Jaehoon Lee" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "kernel-closed-form" ], "facets": { "parameterization": [ "pixels" ], "labels": "mixed", "prior": [ "ntk-kernel" ], "regime": "bilevel", "scale": [ "mnist-cifar" ] }, "tier": "landmark", "links": { "code": "https://github.com/google/neural-tangents", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Introduces Kernel Inducing Points: treats dataset distillation as meta-learning support points for kernel ridge regression under the infinite-width neural-tangent-kernel correspondence, giving a closed-form inner solution and compressing MNIST and CIFAR-10 by one to two orders of magnitude while the resulting synthetic data remains usable to train finite-width networks.", "abstract": "One of the most fundamental aspects of any machine learning algorithm is the training data used by the algorithm. We introduce the novel concept of $\\epsilon$-approximation of datasets, obtaining datasets which are much smaller than or are significant corruptions of the original training data while maintaining similar model performance. We introduce a meta-learning algorithm called Kernel Inducing Points (KIP) for obtaining such remarkable datasets, inspired by the recent developments in the correspondence between infinitely-wide neural networks and kernel ridge-regression (KRR). For KRR tasks, we demonstrate that KIP can compress datasets by one or two orders of magnitude, significantly improving previous dataset distillation and subset selection methods while obtaining state of the art results for MNIST and CIFAR-10 classification. Furthermore, our KIP-learned datasets are transferable to the training of finite-width neural networks even beyond the lazy-training regime, which leads to state of the art results for neural network dataset distillation with potential applications to privacy-preservation.", "awesome_sections": [ "Main / Kernel-Based Distillation" ], "explained": { "before": "Responds to coreset/subset-selection methods (e.g. SVM support vectors, Borsos et al.'s bilevel coresets), which need asymptotically many points, and to bi-level, gradient-based dataset distillation (dd-2018, and gradient-matching's dc-2021 as a contemporaneous baseline), which needs second-order optimization through an iterative inner learner. Also positions itself against label-only distillation (SLDD, LD) as a narrower special case of what it can do.", "problem": "Diagnoses that optimizing a synthetic dataset for a general learning algorithm requires differentiating through that algorithm's (iterative) inner training loop, which is expensive and second-order; the paper asks whether an inner learner can be chosen so the outer optimization becomes first-order and closed-form, and formalizes \"how small can an approximating dataset be\" as epsilon-approximation.", "idea": "Kernel ridge regression has a closed-form solution, and the neural tangent kernel (NTK) makes KRR a proxy for training an infinitely wide neural network; so treat the small \"support\" set fed to KRR as the thing being learned (Kernel Inducing Points), backpropagating only through the closed-form KRR loss rather than through any iterative training.", "method": "Optimizes support images $X_s$ (and optionally support labels $y_s$, or labels alone via the closed-form Label Solve) to minimize the KRR loss $L(X_s,y_s)=\\tfrac12\\|y_t-K_{X_tX_s}(K_{X_sX_s}+\\lambda I)^{-1}y_s\\|^2$ evaluated against real target batches $(X_t,y_t)$, where $K$ is the NTK (or NNGP) of a fixed architecture (FC1 or shallow Myrtle conv nets); a random kernel is sampled per step from a family (FC/Conv, depths 1-3) so the learned set is not overfit to one kernel. Nothing about a network's weights is stored or trained end-to-end; the only artifact is the support set itself. Label Solve (LS) instead fixes $X_s$ and solves for the minimum-norm $y_s$ in closed form via a pseudo-inverse, requiring no gradient descent at all for the label channel.", "evidence": "MNIST KRR, RBF kernel, IPC-equivalent support sizes 10/100/500/5000/10000 images: 89.6/97.3/98.3/ 98.7/98.7% (Table 1); with augmentation+learned labels (\"a+l\"), 90.6/97.8/98.9/99.3/99.3%, beating contemporaneous LeNet-based DD (79.5% at 100 images) and DC (93.9% at 100 images) despite using a much shallower FC1/RBF kernel. CIFAR-10 KRR, RBF (a+l): 40.3/53.8/60.1/65.6/66.3% at the same budgets (Table 2), and FC1-trained neural networks on 5000-10000 KIP images reach 50.1/51.7% versus AlexNet trained via DC (39.1% at 100) or DD (36.8% at 100) -- a shallow FC1 network on KIP data beats deeper architectures trained with prior distillation methods. Cross-kernel transfer (Sec 4.2): datasets trained on a random mixture of six FC/Conv kernels (depths 1-3) generalize better on average than single-kernel training; Label-Solve labels transfer between FC1 and Myrtle-10 kernels with only negligible accuracy loss. Cross-architecture / out-of-distribution transfer to neural networks whose kernel was never sampled during KIP training (LeNet, Wide ResNet) still beats natural images but with a smaller margin than in-distribution transfer. Corruption/privacy ablation: 90%-pixel-corrupted KIP images at 10K support size reach 49.9% on CIFAR-10 (FC1), exceeding even uncorrupted natural images of the same size (45.5%). No GPU-hour or wall-clock cost figure is reported; the paper instead notes that deeper/more diverse architectures were infeasible given compute limits, restricting experiments to shallow kernels (FC1, Myrtle-5/10).", "limitations": "States explicitly that computational limits forced the use of the simplest kernels (FC1, shallow Myrtle nets) rather than deeper architectures like LeNet/AlexNet/ConvNet, leaving open whether gains persist at depth (frepo-2022 and rfad-2022 later address exactly this). The NTK-KRR proxy is exact only for infinite width, so a lazy-training gap opens when the learned data trains finite networks -- observed directly as weaker (though still positive) transfer to out-of-distribution architectures (LeNet, Wide ResNet) whose kernel was not sampled during optimization. Learning labels jointly with images (\"a+l\") sometimes hurts at very large support sizes, which the authors attribute to label overfitting to KRR. Evaluated only on MNIST, Fashion-MNIST and CIFAR-10; no CIFAR-100, Tiny-ImageNet or ImageNet-1K result, and the kernel Gram matrix used in the loss scales with support-set size, limiting how large a set can be optimized in practice." } }, { "id": "new-properties-of-the-2020", "type": "paper", "title": "New Properties of the Data Distillation Method When Working With Tabular Data", "arxiv": "2010.09839", "url": null, "date": "2020-10-19", "venue": { "name": "AIST 2020", "status": "conference" }, "authors": [ "Dmitry Medvedev", "Alexander D'yakonov" ], "kind": "analysis-theory", "settings": [ "other-data" ], "families": [ "meta-learning-bptt" ], "tier": "catalogued", "links": { "code": "https://github.com/dm-medvedev/dataset-distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "This paper applies the original backpropagation-through-time dataset distillation method to tabular data and finds that a model trained on the distilled samples can outperform one trained on the original dataset, but that the distilled data generalizes poorly to models with different hyperparameters than the one used during distillation. It shows that distilling with multiple architectures simultaneously mitigates this poor cross-hyperparameter generalization.", "abstract": "Data distillation is the problem of reducing the volume oftraining data while keeping only the necessary information. With thispaper, we deeper explore the new data distillation algorithm, previouslydesigned for image data. Our experiments with tabular data show thatthe model trained on distilled samples can outperform the model trainedon the original dataset. One of the problems of the considered algorithmis that produced data has poor generalization on models with differenthyperparameters. We show that using multiple architectures during distillation can help overcome this problem.", "notes": "" }, { "id": "distilled-one-shot-federated-2020", "type": "paper", "title": "Distilled One-Shot Federated Learning", "short": "DOSFL", "arxiv": "2009.07999", "url": null, "date": "2020-09-17", "venue": { "name": "arXiv 2020", "status": "preprint" }, "authors": [ "Yanlin Zhou", "George Pu", "Xiyao Ma", "Xiaolin Li", "Dapeng Wu" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "meta-learning-bptt" ], "applications": [ "federated-learning", "privacy" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Each client distills its private dataset into synthetic data and uploads only that in a single round, training a global model without exchanging model weights or gradients; the distilled data is reported useless to anyone without the exact initial model weights, since it is tied to the specific training trajectory it was distilled for. Reports up to three orders of magnitude less communication than FedAvg while retaining 93-99% of centralized-training performance across vision and language tasks.", "abstract": "Current federated learning algorithms take tens of communication rounds transmitting unwieldy model weights under ideal circumstances and hundreds when data is poorly distributed. Inspired by recent work on dataset distillation and distributed one-shot learning, we propose Distilled One-Shot Federated Learning (DOSFL) to significantly reduce the communication cost while achieving comparable performance. In just one round, each client distills their private dataset, sends the synthetic data (e.g. images or sentences) to the server, and collectively trains a global model. The distilled data look like noise and are only useful to the specific model weights, i.e., become useless after the model updates. With this weight-less and gradient-less design, the total communication cost of DOSFL is up to three orders of magnitude less than FedAvg while preserving between 93% to 99% performance of a centralized counterpart. Afterwards, clients could switch to traditional methods such as FedAvg to finetune the last few percent to fit personalized local models with local datasets. Through comprehensive experiments, we show the accuracy and communication performance of DOSFL on both vision and language tasks with different models including CNN, LSTM, Transformer, etc. We demonstrate that an eavesdropping attacker cannot properly train a good model using the leaked distilled data, without knowing the initial model weights. DOSFL serves as an inexpensive method to quickly converge on a performant pre-trained model with less than 0.1% communication cost of traditional methods.", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "" }, { "id": "federated-learning-via-synthetic-2020", "type": "paper", "title": "Federated Learning via Synthetic Data", "arxiv": "2008.04489", "url": null, "date": "2020-08-11", "venue": { "name": "arXiv 2020", "status": "preprint" }, "authors": [ "Jack Goetz", "Ambuj Tewari" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "meta-learning-bptt" ], "applications": [ "federated-learning" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Proposes transmitting a small amount of synthetic data from client to server in place of a gradient update, avoiding the multi-million-parameter cost of standard federated learning updates. Reports more than an order-of-magnitude reduction in communication cost with minimal model degradation in early experiments.", "abstract": "Federated learning allows for the training of a model using data on multiple clients without the clients transmitting that raw data. However the standard method is to transmit model parameters (or updates), which for modern neural networks can be on the scale of millions of parameters, inflicting significant computational costs on the clients. We propose a method for federated learning where instead of transmitting a gradient update back to the server, we instead transmit a small amount of synthetic `data'. We describe the procedure and show some experimental results suggesting this procedure has potential, providing more than an order of magnitude reduction in communication costs with minimal model degradation.", "awesome_sections": [ "Applications / Federated Learning" ], "notes": "" }, { "id": "label-distillation-2020", "type": "paper", "title": "Flexible Dataset Distillation: Learn Labels Instead of Images", "short": "Learned labels", "arxiv": "2006.08572", "url": null, "date": "2020-06-15", "venue": { "name": "NeurIPS 2020 Workshop", "status": "workshop" }, "authors": [ "Ondrej Bohdal", "Yongxin Yang", "Timothy Hospedales" ], "kind": "method", "settings": [ "image-classification", "text" ], "families": [ "label-distillation" ], "tier": "catalogued", "links": { "code": "https://github.com/ondrejbohdal/label-distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Studies label distillation -- learning synthetic soft labels for a small, fixed set of real images -- as an alternative to distilling images, using a more robust meta-learning algorithm and a first-order strategy based on convex optimization layers; shows label distillation improves on prior image-based distillation, generalizes across optimizers and architectures more readily, and can even transfer across datasets, e.g. learning Japanese character recognition from synthetically labeled English letters.", "abstract": "We study the problem of dataset distillation - creating a small set of synthetic examples capable of training a good model. In particular, we study the problem of label distillation - creating synthetic labels for a small set of real images, and show it to be more effective than the prior image-based approach to dataset distillation. Methodologically, we introduce a more robust and flexible meta-learning algorithm for distillation, as well as an effective first-order strategy based on convex optimization layers. Distilling labels with our new algorithm leads to improved results over prior image-based distillation. More importantly, it leads to clear improvements in flexibility of the distilled dataset in terms of compatibility with off-the-shelf optimizers and diverse neural architectures. Interestingly, label distillation can also be applied across datasets, for example enabling learning Japanese character recognition by training only on synthetically labeled English letters.", "awesome_sections": [ "Main / Label Distillation" ] }, { "id": "dc-2021", "type": "paper", "title": "Dataset Condensation with Gradient Matching", "short": "DC", "arxiv": "2006.05929", "url": null, "date": "2020-06-10", "venue": { "name": "ICLR 2021", "status": "conference" }, "authors": [ "Bo Zhao", "Konda Reddy Mopuri", "Hakan Bilen" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "gradient-matching" ], "facets": { "parameterization": [ "pixels" ], "labels": "hard", "prior": [ "random-networks" ], "regime": "single-level", "scale": [ "mnist-cifar" ] }, "tier": "landmark", "links": { "code": "https://github.com/VICO-UoE/DatasetCondensation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Introduces dataset condensation as gradient matching: at every training step, match the gradients a network computes on a small synthetic set to the gradients it computes on the corresponding class of real data, learning the synthetic images by backpropagating through this per-step matching loss rather than unrolling full training.", "abstract": "As the state-of-the-art machine learning methods in many fields rely on larger datasets, storing datasets and training models on them become significantly more expensive. This paper proposes a training set synthesis technique for data-efficient learning, called Dataset Condensation, that learns to condense large dataset into a small set of informative synthetic samples for training deep neural networks from scratch. We formulate this goal as a gradient matching problem between the gradients of deep neural network weights that are trained on the original and our synthetic data. We rigorously evaluate its performance in several computer vision benchmarks and demonstrate that it significantly outperforms the state-of-the-art methods. Finally we explore the use of our method in continual learning and neural architecture search and report promising gains when limited memory and computations are available.", "awesome_sections": [ "Main / Gradient/Trajectory Matching Surrogate Objective" ], "explained": { "before": "Responds to dd-2018's bi-level backprop-through-training, which the paper frames as expensive and hard to scale because it must unroll and differentiate through many training steps, and to coreset-selection methods (random, herding, k-center, forgetting), which are cheap but limited to real images and shown to plateau well below synthesized data at the same budget.", "problem": "Bi-level dataset distillation (dd-2018) requires backpropagating through an entire (possibly multi-step) inner training procedure to know how changing the synthetic images affects final model performance -- expensive, higher-order, and hard to scale to more steps or larger networks. The paper asks whether a cheaper, local proxy for \"trains like the real data\" exists.", "idea": "Instead of matching final trained performance (which requires unrolling many steps), match the gradient a network computes on the synthetic set to the gradient it computes on the real set, at every step of an ordinary training trajectory -- if the two gradients agree at each step, the parameter trajectories induced by training on either set will stay close, without ever backpropagating through more than one step.", "method": "Optimizes a small per-class synthetic image set (pixels only, fixed hard labels). In the loop: initialize a network from a random $\\theta_0\\sim P_{\\theta_0}$, and at each of T outer steps compute the classification-loss gradient $\\nabla_\\theta\\mathcal{L}^{\\mathcal{S}}(\\theta_t)$ on a synthetic class-batch and $\\nabla_\\theta\\mathcal{L}^{\\mathcal{T}}(\\theta_t)$ on the corresponding real class-batch, then minimize their distance $D(\\cdot,\\cdot)$ (Eq. 9) by updating the synthetic images via one SGD step; periodically update $\\theta_t$ itself by training on the synthetic images. The distance is a layer-wise, per-output-node cosine distance (Eq. 10) -- gradients are grouped by output node rather than flattened across the whole network -- which the paper finds is the effective choice (vs. a single flattened Euclidean or Cosine distance over all parameters). The whole procedure is repeated over many random initializations $\\theta_0$ so the synthetic set is not tied to one network.", "evidence": "Table 1 (ConvNet, hard labels, coreset baselines vs. Ours vs. whole dataset): MNIST IPC1/10/50 = 91.7+/-0.5 / 97.4+/-0.2 / 98.8+/-0.2% (full 99.6+/-0.0%); FashionMNIST IPC1/10/50 = 70.5+/-0.6 / 82.3+/-0.4 / 83.6+/-0.4% (full 93.5+/-0.1%); SVHN IPC1/10/50 = 31.2+/-1.4 / 76.1+/-0.6 / 82.3+/-0.3% (full 95.4+/-0.1%); CIFAR-10 IPC1/10/50 = 28.3+/-0.5 / 44.9+/-0.5 / 53.9+/-0.5% (full 84.8+/-0.1%), beating random/herding/k-center/forgetting coreset selection at every IPC on every dataset. CIFAR-100 (Table T20, appendix): IPC1/10 = 12.8+/-0.3 / 25.2+/-0.3% (full 56.2+/-0.3%); no IPC50 reported. Cost (Table T22): on MNIST/LeNet, DD uses 785MB/160min for 79.5+/-8.1% vs. DC's 653MB/46min for 93.9+/-0.6%; on CIFAR-10/AlexCifarNet, DD uses 3211MB/214min for 36.8+/-1.2% vs. DC's 1445MB/105min for 39.1+/-1.2% -- less memory, less time, and higher accuracy than dd-2018's bi-level BPTT at the same budget. Ablation (Table T18, MNIST IPC1, matched train/test architecture): the layer-wise per-output-node cosine distance (\"Ours\") reaches 70.5/91.7/85.0/82.7/81.7/89.4% on MLP/ConvNet/LeNet/AlexNet/VGG/ResNet vs. a flattened whole-network Euclidean distance's 69.3/92.7/65.0/66.2/57.1/68.0% and a flattened Cosine distance's 45.2/69.2/61.1/58.3/55.0/68.8% -- the layer-wise grouping (not cosine similarity alone) is what keeps accuracy high on deeper architectures (LeNet/AlexNet/VGG/ResNet), where flattened distances degrade sharply.", "limitations": "Does not use data augmentation during matching (identified and fixed by dsa-2021, its direct successor); only reports IPC 1/10/50 on MNIST/FashionMNIST/SVHN/CIFAR-10 and IPC 1/10 (no 50) on CIFAR-100, with no Tiny-ImageNet or ImageNet-scale result. The single-step gradient-matching signal is short-horizon by construction -- it only guarantees local agreement between synthetic- and real-data gradients at sampled points along a trajectory, not that the trajectories stay close over many steps, a gap dcc-2022 shows can make it underperform random selection on fine-grained data and that mtt-2022 addresses directly by matching multi-step trajectories instead." } }, { "id": "reducing-catastrophic-forgetting-with-2020", "type": "paper", "title": "Reducing catastrophic forgetting with learning on synthetic data", "arxiv": "2004.14046", "url": null, "date": "2020-04-29", "venue": { "name": "CVPR 2020 Workshop", "status": "workshop" }, "authors": [ "Wojciech Masarczyk", "Ivona Tautkute" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "meta-learning-bptt" ], "applications": [ "continual-learning" ], "tier": "catalogued", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Generates synthetic replay data via a two-step meta-gradient optimization process specifically so that learning tasks on it in sequence does not cause catastrophic forgetting. Reports that training on such synthetic data in sequence avoids catastrophic forgetting on Split-MNIST, and that the generation method is robust across different learning scenarios.", "abstract": "Catastrophic forgetting is a problem caused by neural networks' inability to learn data in sequence. After learning two tasks in sequence, performance on the first one drops significantly. This is a serious disadvantage that prevents many deep learning applications to real-life problems where not all object classes are known beforehand; or change in data requires adjustments to the model. To reduce this problem we investigate the use of synthetic data, namely we answer a question: Is it possible to generate such data synthetically which learned in sequence does not result in catastrophic forgetting? We propose a method to generate such data in two-step optimisation process via meta-gradients. Our experimental results on Split-MNIST dataset show that training a model on such synthetic data in sequence does not result in catastrophic forgetting. We also show that our method of generating data is robust to different learning scenarios.", "awesome_sections": [ "Applications / Continual Learning" ], "notes": "" }, { "id": "gtn-2020", "type": "paper", "title": "Generative Teaching Networks: Accelerating Neural Architecture Search by Learning to Generate Synthetic Training Data", "short": "GTN", "arxiv": "1912.07768", "url": null, "date": "2019-12-17", "venue": { "name": "ICML 2020", "status": "conference" }, "authors": [ "Felipe Petroski Such", "Aditya Rawal", "Joel Lehman", "Kenneth O. Stanley", "Jeff Clune" ], "kind": "application", "settings": [ "image-classification" ], "families": [ "generator-as-dataset", "meta-learning-bptt" ], "applications": [ "nas" ], "tier": "notable", "links": { "code": "https://github.com/uber-research/GTN", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Generative Teaching Networks train a generator network to produce synthetic training data (and, in principle, training environments or curricula) on which a freshly initialized learner takes a few SGD steps before being evaluated on the target task, with the generator's parameters updated by differentiating through that whole short training process via meta-gradients. Used to speed up architecture evaluation in neural architecture search, GTN-NAS is reported to improve on the NAS state of the art for a given search proposal mechanism while using orders of magnitude less computation than typical NAS methods.", "abstract": "This paper investigates the intriguing question of whether we can create learning algorithms that automatically generate training data, learning environments, and curricula in order to help AI agents rapidly learn. We show that such algorithms are possible via Generative Teaching Networks (GTNs), a general approach that is, in theory, applicable to supervised, unsupervised, and reinforcement learning, although our experiments only focus on the supervised case. GTNs are deep neural networks that generate data and/or training environments that a learner (e.g. a freshly initialized neural network) trains on for a few SGD steps before being tested on a target task. We then differentiate through the entire learning process via meta-gradients to update the GTN parameters to improve performance on the target task. GTNs have the beneficial property that they can theoretically generate any type of data or training environment, making their potential impact large. This paper introduces GTNs, discusses their potential, and showcases that they can substantially accelerate learning. We also demonstrate a practical and exciting application of GTNs: accelerating the evaluation of candidate architectures for neural architecture search (NAS), which is rate-limited by such evaluations, enabling massive speed-ups in NAS. GTN-NAS improves the NAS state of the art, finding higher performing architectures when controlling for the search proposal mechanism. GTN-NAS also is competitive with the overall state of the art approaches, which achieve top performance while using orders of magnitude less computation than typical NAS methods. Speculating forward, GTNs may represent a first step toward the ambitious goal of algorithms that generate their own training data and, in doing so, open a variety of interesting new research questions and directions.", "awesome_sections": [ "Applications / Neural Architecture Search" ], "notes": "" }, { "id": "optimizing-millions-of-hyperparameters-2020", "type": "paper", "title": "Optimizing Millions of Hyperparameters by Implicit Differentiation", "arxiv": "1911.02590", "url": null, "date": "2019-11-06", "venue": { "name": "AISTATS 2020", "status": "conference" }, "authors": [ "Jonathan Lorraine", "Paul Vicol", "David Duvenaud" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [], "tier": "catalogued", "links": { "code": "https://github.com/MaximeVandegar/Papers-in-100-Lines-of-Code/tree/main/Optimizing_Millions_of_Hyperparameters_by_Implicit_Differentiation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Proposes an implicit-function-theorem approach to hyperparameter gradients using efficient inverse-Hessian approximations, enabling joint optimization of millions of weights and hyperparameters (including a learned data-augmentation network) at only a few times the cost of standard training — an implicit-differentiation technique later adopted by kernel-based dataset distillation methods to avoid unrolling the inner loop.", "abstract": "We propose an algorithm for inexpensive gradient-based hyperparameter optimization that combines the implicit function theorem (IFT) with efficient inverse Hessian approximations. We present results about the relationship between the IFT and differentiating through optimization, motivating our algorithm. We use the proposed approach to train modern network architectures with millions of weights and millions of hyper-parameters. For example, we learn a data-augmentation network - where every weight is a hyperparameter tuned for validation performance - outputting augmented training examples. Jointly tuning weights and hyperparameters with our approach is only a few times more costly in memory and compute than standard training.", "awesome_sections": [ "Main / Better Understanding" ] }, { "id": "sldd-2021", "type": "paper", "title": "Soft-Label Dataset Distillation and Text Dataset Distillation", "short": "Soft-label DD", "arxiv": "1910.02551", "url": null, "date": "2019-10-06", "venue": { "name": "IJCNN 2021", "status": "conference" }, "authors": [ "Ilia Sucholutsky", "Matthias Schonlau" ], "kind": "method", "settings": [ "image-classification", "text" ], "families": [ "label-distillation" ], "tier": "catalogued", "links": { "code": "https://github.com/ilia10000/dataset-distillation", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Extends dataset distillation with soft (distributional) labels learned jointly with the synthetic images instead of one hard label per sample, letting a single synthetic sample encode information for multiple classes; reports 2-4% accuracy gains over hard-label distillation, over 96% MNIST accuracy from 10 distilled images and about 92% from just 5, and extends the same approach to distilling sequential text data, retaining near-original accuracy on IMDB sentiment analysis from 20 distilled sentences.", "abstract": "Dataset distillation is a method for reducing dataset sizes by learning a small number of synthetic samples containing all the information of a large dataset. This has several benefits like speeding up model training, reducing energy consumption, and reducing required storage space. Currently, each synthetic sample is assigned a single `hard' label, and also, dataset distillation can currently only be used with image data. We propose to simultaneously distill both images and their labels, thus assigning each synthetic sample a `soft' label (a distribution of labels). Our algorithm increases accuracy by 2-4% over the original algorithm for several image classification tasks. Using `soft' labels also enables distilled datasets to consist of fewer samples than there are classes as each sample can encode information for multiple classes. For example, training a LeNet model with 10 distilled images (one per class) results in over 96% accuracy on MNIST, and almost 92% accuracy when trained on just 5 distilled images. We also extend the dataset distillation algorithm to distill sequential datasets including texts. We demonstrate that text distillation outperforms other methods across multiple datasets. For example, models attain almost their original accuracy on the IMDB sentiment analysis task using just 20 distilled sentences. Our code can be found at $\\href{https://github.com/ilia10000/dataset-distillation}{\\text{https://github.com/ilia10000/dataset-distillation}}$.", "awesome_sections": [ "Main / Label Distillation" ] }, { "id": "forgetting-2019", "type": "paper", "title": "An Empirical Study of Example Forgetting during Deep Neural Network Learning", "short": "Forgetting", "arxiv": "1812.05159", "url": null, "date": "2018-12-12", "venue": { "name": "ICLR 2019", "status": "conference" }, "authors": [ "Mariya Toneva", "Alessandro Sordoni", "Remi Tachet des Combes", "Adam Trischler", "Yoshua Bengio", "Geoffrey J. Gordon" ], "kind": "baseline", "settings": [ "image-classification" ], "families": [ "coreset-selection" ], "tier": "notable", "links": { "code": null, "project": null }, "facets": { "labels": "hard", "regime": "selection", "scale": [ "mnist-cifar" ] }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "Curriculum learning (Bengio et al. 2009) and self-paced learning (Kumar et al. 2010) define example \"hardness\" from the loss at one point in training; Chang et al. (2017) re-weight examples by the variance of their predictive distribution but do not analyze how often this occurs; Koh and Liang's influence functions and various importance-sampling schemes (Zhao and Zhang; Katharopoulos and Fleuret) score examples from gradient norms. None of these track whether an example, once correctly classified during training, is later misclassified again — the paper's own definition of \"forgetting\" — nor ask whether removing a data-derived subset preserves generalization.", "problem": "It is unknown whether \"catastrophic forgetting\" (studied in continual learning as a consequence of a distribution shift between sequential tasks) also occurs for individual examples within a single, stationary classification task trained by ordinary SGD, and if so, whether that forgetting behavior reveals which training examples are redundant and safe to discard.", "idea": "Track each training example's correct/incorrect classification state across all SGD updates where it appears in a minibatch; call a transition from correct to incorrect a \"forgetting event.\" Some examples are forgotten repeatedly, others (\"unforgettable\") never are once first learned. Forgetting frequency is stable across seeds and architectures and correlates with an example's intrinsic difficulty (noisy labels, atypical features), so unforgettable examples are redundant and can be removed from training without hurting generalization, while highly forgotten examples behave like support vectors.", "method": "For each example $i$ and SGD step $t$, define $\\text{acc}_i^t=\\mathbb{1}[\\hat{y}_i^t=y_i]$; a forgetting event occurs when $\\text{acc}_i^t>\\text{acc}_i^{t+1}$ (correct then incorrect) and a learning event when the reverse holds. Forgetting counts are accumulated only across the presentations of an example within its sampled minibatches (a lower bound on the true event count, Algorithm 1), avoiding the cost of scoring the whole dataset every step. An example is \"unforgettable\" if it is learned at some finite step and never misclassified again. The resulting per-example forgetting count is used to rank and prune the training set: examples are removed in increasing order of forgetting count (least-forgotten/most-redundant first) and a fresh model is retrained from scratch on each pruned subset. No teacher, generator, or bi-level optimization is involved; scoring requires one full training run of the same (or a smaller/cheaper) network on the full dataset. In the loop: only the network being scored (a 2-conv-layer net for MNIST/ permutedMNIST, a ResNet-18 with cutout for CIFAR-10/100, and separately a smaller CNN and a WideResNet used to test whether the ranking transfers across architectures).", "evidence": "CIFAR-10, hard labels, ResNet-18 (Fig. 5, left): removing examples in increasing order of forgetting count allows 30% of the training set to be removed with performance comparable to the full-data model, and up to 35% removed with less than 0.2 percentage points of degradation, while removing a random 30% subset degrades accuracy much faster. Cross-dataset removal budgets (Fig. 6): up to 30% removable on CIFAR-10, 50% on permutedMNIST, 80% on MNIST, tracking each dataset's fraction of unforgettable examples (31.3% CIFAR-10, 75.3% permutedMNIST, 91.7% MNIST, all measured across 5 seeds). CIFAR-100 (Appendix 15): only 7.62% of examples are unforgettable and about 8% of the training set can be removed losslessly — far less headroom than CIFAR-10, tracking the harder, more fine-grained task. Stability: average pairwise Pearson correlation of per-example forgetting counts across 10 seeds is 89.2%, rising to 97.6% when comparing sums over two groups of 5 seeds; \"chance\" forgetting under randomized (shuffled) gradient updates occurs at most twice per example, versus the true distribution's much longer tail. Cross-architecture transfer (Fig. 7): an ordering computed from a much smaller 2-conv-layer network recovers ResNet-18's unforgettable examples with high precision/recall, and pruning a WideResNet's training set by 30% using a ResNet-18-derived ordering (2 GPU-hours to compute vs. 8 GPU-hours for the WideResNet itself, saving up to 6 hours) still yields near-optimal WideResNet accuracy. Noisy-label diagnostic (Fig. 3): with 20% of CIFAR-10 labels randomized, the most-forgotten examples are overwhelmingly the noisy ones, and no noisy example is unforgettable.", "limitations": "Restricted to MNIST, permutedMNIST, CIFAR-10 and CIFAR-100 — no ImageNet-scale or modern architecture evaluation, and no comparison to any dataset-distillation method (predates the field's 2019-onward coreset/DD comparisons). Computing exact forgetting statistics requires one full training run per architecture, which the paper itself calls \"computationally expensive,\" motivating its own cross-architecture-transfer experiment as a workaround rather than a solution. The ranking stabilizes only after roughly 50-75 epochs of the scoring run (Fig. 7 left), so early-stopping the scoring pass is not free. At the most aggressive pruning ratios, the most-forgotten examples themselves start to hurt generalization (Fig. 5, right, upturn at the tail), which the paper attributes to outliers/mislabeled data without a method to separate them from genuinely informative hard examples." }, "summary": "Defines a 'forgetting event' as a training example transitioning from correctly to incorrectly classified over the course of training, and studies these events across benchmark datasets. Reports that some examples are forgotten frequently and others never, that a dataset's unforgettable examples generalize across architectures, and that a significant fraction of examples can be removed from training, ranked by forgetting frequency, without hurting generalization — the basis of the forgetting-score coreset baseline.", "abstract": "Inspired by the phenomenon of catastrophic forgetting, we investigate the learning dynamics of neural networks as they train on single classification tasks. Our goal is to understand whether a related phenomenon occurs when data does not undergo a clear distributional shift. We define a `forgetting event' to have occurred when an individual training example transitions from being classified correctly to incorrectly over the course of learning. Across several benchmark data sets, we find that: (i) certain examples are forgotten with high frequency, and some not at all; (ii) a data set's (un)forgettable examples generalize across neural architectures; and (iii) based on forgetting dynamics, a significant fraction of examples can be omitted from the training data set while still maintaining state-of-the-art generalization performance.", "notes": "" }, { "id": "dd-2018", "type": "paper", "title": "Dataset Distillation", "short": "DD", "arxiv": "1811.10959", "url": null, "date": "2018-11-27", "venue": { "name": "arXiv 2018", "status": "preprint" }, "authors": [ "Tongzhou Wang", "Jun-Yan Zhu", "Antonio Torralba", "Alexei A. Efros" ], "kind": "method", "settings": [ "image-classification" ], "families": [ "meta-learning-bptt" ], "facets": { "parameterization": [ "pixels" ], "labels": "hard", "prior": [ "none", "pretrained-classifier" ], "regime": "bilevel", "scale": [ "mnist-cifar" ] }, "tier": "landmark", "links": { "code": "https://github.com/SsnL/dataset-distillation", "project": "https://ssnl.github.io/dataset_distillation/" }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "summary": "Introduces dataset distillation: a small set of synthetic training images, not necessarily on the real-data manifold, optimized by backpropagating through several steps of gradient descent so that training a fixed-initialization network on them approximates training on the full dataset, compressing 60,000 MNIST images into 10 synthetic images with close to the original accuracy.", "abstract": "Model distillation aims to distill the knowledge of a complex model into a simpler one. In this paper, we consider an alternative formulation called dataset distillation: we keep the model fixed and instead attempt to distill the knowledge from a large training dataset into a small one. The idea is to synthesize a small number of data points that do not need to come from the correct data distribution, but will, when given to the learning algorithm as training data, approximate the model trained on the original data. For example, we show that it is possible to compress 60,000 MNIST training images into just 10 synthetic distilled images (one per class) and achieve close to original performance with only a few gradient descent steps, given a fixed network initialization. We evaluate our method in various initialization settings and with different learning objectives. Experiments on multiple datasets show the advantage of our approach compared to alternative methods.", "awesome_sections": [ "Main" ], "explained": { "before": "Responds to network/model distillation (Hinton et al., 2015), which compresses a trained model rather than a dataset, and to coreset construction, instance selection and active-learning subset methods, which select real \"valuable\" examples rather than synthesize new ones and therefore need far more than one example per class. Also related to, but going further than, gradient-based hyperparameter optimization (Maclaurin et al. 2015), which backpropagates through training to tune hyperparameters rather than the training data itself.", "problem": "Asks how much information a training set actually contains and whether that information can be repackaged as a much smaller synthetic set. Standard training needs tens of thousands to millions of gradient steps over the full data; the paper's diagnosis is that nothing in principle requires the training examples themselves, rather than some other small set of images, to be the vehicle for that information.", "idea": "Treat the pixels of a handful of synthetic images (and the learning rate) as the parameters being optimized, and define their loss as the real-data loss of a network after it takes a few gradient steps on them starting from a given initialization. Because that composed function is differentiable end to end, the synthetic images can be learned by backpropagating through the inner gradient steps.", "method": "Optimizes M synthetic images $\\tilde{\\mathbf{x}}$ and per-step learning rates $\\tilde{\\eta}$; the network architecture is fixed and untrained. For a fixed initialization $\\theta_0$, one or more GD steps on $\\tilde{\\mathbf{x}}$ produce $\\theta_1,\\theta_2,\\dots$, and the outer objective is the real training-data loss $\\ell(\\mathbf{x},\\theta_T)$, minimized over $\\tilde{\\mathbf{x}},\\tilde{\\eta}$ by backpropagating through the T inner steps. For random initializations the objective is the expectation over $\\theta_0\\sim p(\\theta_0)$ (Eq. 4), so the same synthetic set must work for many sampled networks (4-16 samples per outer step). Multi-step, multi-epoch unrolling is made tractable with back-gradient optimization (Hessian-vector products via Pearlmutter's trick) instead of naive backprop-through-time. Class labels are fixed (hard), not optimized. A linear-regression special case (Sec. 3.3) shows that for one GD step and arbitrary initialization, at least M >= D synthetic points are needed, where D is the input dimension.", "evidence": "MNIST/LeNet, fixed init, 10 images total (1/class), 10 GD steps x 3 epochs: 12.90% -> 93.76% test accuracy vs. 99% fully trained (Fig. 2a); CIFAR-10, fixed init, 100 images (10/class): 8.82% -> 54.03% vs. 80% fully trained (Fig. 2b, Table 1). Random init, 100 images: MNIST 79.5+/-8.1%, CIFAR-10 36.8+/-1.2% (Table 1), beating k-means, average-real, random-real and optimized-real-image baselines at the same image budget. Cross-domain adaptation (100 images) among MNIST/USPS/SVHN beats a few-shot domain-adaptation baseline (Motiian et al. 2017) on 2/3 transfers (Table 2). Fixed ImageNet-pretrained AlexNet adapted with 1 image/class: PASCAL-VOC 70.75% (vs. 19.41% random-real, 75.57% full fine-tune) and CUB-200 38.76% (vs. 7.11% random-real, 41.21% full fine-tune) (Table 3). Ablation: at matched total image budget, multiple GD steps drastically outperform a single step with a larger batch (Fig. 5), and more epochs help but saturate (Fig. 4) -- isolating that spreading a fixed budget over more optimization steps, not just more pixels, is what makes the images informative. Cost: each training run took 1-4 hours on a single Titan Xp/V100 GPU (fixed init) or 4 GPUs (random init, more initializations sampled per step); no epoch/wall-clock comparison to full-data training is given.", "limitations": "States it is sensitive to the distribution of initializations p(theta_0) and only tests MNIST, CIFAR-10 and small transfer-learning benchmarks (PASCAL-VOC, CUB-200) at low resolution -- no results at CIFAR-100/Tiny-ImageNet/ImageNet scale. Labels are hard and tied to a fixed class assignment, not learned. The linear-case analysis shows a hard lower bound (M >= input dimension D) for one-step, arbitrary-initialization distillation, foreshadowing why later work moved to random/expert-trajectory ensembles or closed-form inner solvers. Backpropagating through even a handful of inner steps requires second-order (Hessian-vector) computation, which does not scale to the many steps or large architectures later families needed -- the scaling problem the family's subsequent papers (kernel-closed-form, gradient-matching, trajectory-matching) were created to avoid." } }, { "id": "kcenter-coreset-2018", "type": "paper", "title": "Active Learning for Convolutional Neural Networks: A Core-Set Approach", "short": "k-Center coreset", "arxiv": "1708.00489", "url": null, "date": "2017-08-01", "venue": { "name": "ICLR 2018", "status": "conference" }, "authors": [ "Ozan Sener", "Silvio Savarese" ], "kind": "baseline", "settings": [ "image-classification" ], "families": [ "coreset-selection" ], "tier": "notable", "links": { "code": null, "project": null }, "facets": { "labels": "hard", "regime": "selection", "scale": [ "mnist-cifar" ] }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "full-text" }, "explained": { "before": "Classical active-learning heuristics assume single-point (b=1) querying: uncertainty-based selection (max-entropy, BALD, Variation Ratios), Bayesian methods using MC-dropout as approximate inference (Gal et al.'s DBAL), and batch methods that combine uncertainty with a diversity term (BMDR, minimizing MMD between the pool and selected points; CEAL for weakly-supervised CNNs). The paper's own empirical study finds these ineffective for CNNs in the batch-query setting required in practice, because querying many points per round (rather than one) correlates the selected samples.", "problem": "Training a CNN to convergence for every single-point query is intractable, so active learning for CNNs must select a batch of points per round; existing batch heuristics still rely on per-point uncertainty scores that become redundant/correlated once many points are chosen at once, and none of them come with a bound relating the selected subset's size and geometry to the resulting model's population risk.", "idea": "Reformulate batch active learning as core-set selection: choose a subset $\\mathbf{s}$ such that a model trained on $\\mathbf{s}$ has population risk close to a model trained on the full pool. Bound the resulting \"core-set loss\" using only the geometric covering radius of $\\mathbf{s}$ over the full dataset in a feature space (not label information), independent of how many labeled points are used; minimizing this bound is exactly the classical k-Center (minimax facility location) problem, solved with a greedy approximation and refined by an exact mixed-integer-program solver.", "method": "Decomposes population risk into generalization error, training error, and a \"core-set loss\" term $|\\frac1n\\sum_i l(x_i,y_i;A_\\mathbf{s}) - \\frac1{|\\mathbf{s}|}\\sum_{j\\in\\mathbf{s}} l(x_j,y_j;A_\\mathbf{s})|$ (Eq. 3-4). Theorem 1 bounds this core-set loss by $\\mathcal{O}(\\delta_\\mathbf{s}) + \\mathcal{O}(1/\\sqrt n)$ under a zero-training-error assumption, where $\\delta_\\mathbf{s}$ is the covering radius of $\\mathbf{s}$ (every dataset point lies within $\\delta_\\mathbf{s}$ of some point in $\\mathbf{s}$), given that the loss is Lipschitz in the input for fixed label/weights; Lemma 1 proves this Lipschitz property for a ReLU/max-pool CNN under an $\\ell_2$ output loss. The practical objective becomes minimizing $\\delta_\\mathbf{s}$, i.e. the k-Center problem $\\min_{\\mathbf{s}^1}\\max_i\\min_{j\\in\\mathbf{s}^0\\cup\\mathbf{s}^1}\\Delta(x_i,x_j)$ with $\\Delta$ the $\\ell_2$ distance between final-fully-connected-layer activations. Since k-Center is NP-hard, a greedy furthest-point algorithm (k-Center-Greedy, Algorithm 1: repeatedly add the point farthest from the current selected set) gives a 2-OPT solution used to initialize an exact refinement — a mixed-integer program (Algorithm 2/Eq. 6, solved with Gurobi via LP-relaxation and branch-and-bound, with binary search over the covering radius $\\delta$) that also tolerates up to $\\Xi$ outlier points not required to be covered. Selection is entirely unsupervised (uses only image features, never labels), applied iteratively over multiple query rounds, with a fresh VGG-16 trained from scratch (RMSProp) after each round, in both fully-supervised (train only on labeled points) and weakly-supervised (Ladder-network semi-supervised training using unlabeled points too) variants. In the loop: only the CNN being actively trained/queried each round (no separate teacher, generator, or bi-level optimization).", "evidence": "CIFAR-10, CIFAR-100 and SVHN, fully- and weakly-supervised active learning (Figs. 3-4, accuracy vs. number of labeled points, averaged over 5 random initial pools): k-Center-Greedy (with the robust MIP refinement) outperforms Random selection, Best Empirical Uncertainty, DBAL (Bayesian MC-dropout), Best Oracle Uncertainty (which even sees the true labels to compute uncertainty), k-Medoids, and BMDR across the labeled-point budget curve, by a large margin in the weakly- supervised setting; the paper attributes weakly-supervised methods' extra edge to better learned feature geometry, which its purely geometric criterion exploits more directly. The method is comparatively less effective on CIFAR-100 than CIFAR-10/SVHN, which the paper attributes directly to its own bound scaling with the number of classes $C$. tSNE qualitative check (Fig. 5): the method's queries evenly cover the feature-space distribution, while the uncertainty-oracle baseline's queries cluster and fail to cover large regions of the space, visually confirming the covering-radius rationale. Exact vs. greedy solver (Fig. 6, Table 1, CIFAR-100, $b{=}5k$, $|\\mathbf{s}^0|{=}10k$): the MIP-refined solution gives a \"small but significant\" accuracy improvement over the 2-OPT greedy solution alone, at a runtime of 360.23s total (104.2s distance matrix + 2s greedy 2-OPT + 244.03s MIP search) versus greedy alone, both tractable at 50k-image scale on a single workstation CPU.", "limitations": "The paper's own theoretical bound scales with the number of classes $C$, and the method is empirically weaker on the 100-class CIFAR-100 than on 10-class CIFAR-10/SVHN — an explicit class-count limitation. Theorem 1's Lipschitz proof (Lemma 1) is derived for an $\\ell_2$ output loss, not the cross-entropy loss actually used in the reported experiments; the paper states its \"theoretical study does not extend to cross-entropy loss\" though experiments suggest the resulting algorithm still works well with it. The zero-training-error assumption underlying Theorem 1 is acknowledged as \"not entirely realistic.\" The exact MIP refinement is not polynomial-time in the worst case (only empirically tractable at the tested 50k-image scale); the method is never tested at ImageNet scale, and incorporating uncertainty information into the otherwise purely geometric criterion is explicitly left as \"an open problem and a fruitful future research direction.\"" }, "summary": "Poses active learning for CNNs as core-set selection: choose a subset such that a model trained on it is competitive with one trained on the full pool, and derives a bound on this gap from the geometry of the selected points. Selects points with a k-Center greedy covering criterion in feature space, used throughout dataset condensation as the covering-based coreset baseline.", "abstract": "Convolutional neural networks (CNNs) have been successfully applied to many recognition and learning tasks using a universal recipe; training a deep model on a very large dataset of supervised examples. However, this approach is rather restrictive in practice since collecting a large set of labeled images is very expensive. One way to ease this problem is coming up with smart ways for choosing images to be labelled from a very large collection (ie. active learning). Our empirical study suggests that many of the active learning heuristics in the literature are not effective when applied to CNNs in batch setting. Inspired by these limitations, we define the problem of active learning as core-set selection, ie. choosing set of points such that a model learned over the selected subset is competitive for the remaining data points. We further present a theoretical result characterizing the performance of any selected subset using the geometry of the datapoints. As an active learning algorithm, we choose the subset which is expected to yield best result according to our characterization. Our experiments show that the proposed method significantly outperforms existing approaches in image classification experiments by a large margin.", "notes": "" }, { "id": "hypergrad-2015", "type": "paper", "title": "Gradient-based Hyperparameter Optimization through Reversible Learning", "short": "Hypergradient", "arxiv": "1502.03492", "url": null, "date": "2015-02-11", "venue": { "name": "ICML 2015", "status": "conference" }, "authors": [ "Dougal Maclaurin", "David Duvenaud", "Ryan P. Adams" ], "kind": "analysis-theory", "settings": [ "image-classification" ], "families": [], "tier": "catalogued", "links": { "code": "https://github.com/HIPS/hypergrad", "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "arxiv-api", "depth": "abstract" }, "summary": "Computes exact gradients of validation performance with respect to thousands of hyperparameters — including a data-augmentation network whose weights are treated as hyperparameters — by exactly reversing SGD-with-momentum dynamics, a hypergradient technique later reused to differentiate through the training procedure in bi-level dataset distillation.", "abstract": "Tuning hyperparameters of learning algorithms is hard because gradients are usually unavailable. We compute exact gradients of cross-validation performance with respect to all hyperparameters by chaining derivatives backwards through the entire training procedure. These gradients allow us to optimize thousands of hyperparameters, including step-size and momentum schedules, weight initialization distributions, richly parameterized regularization schemes, and neural network architectures. We compute hyperparameter gradients by exactly reversing the dynamics of stochastic gradient descent with momentum.", "awesome_sections": [ "Main / Early Work" ] }, { "id": "herding-2009", "type": "paper", "title": "Herding Dynamical Weights to Learn", "short": "Herding", "arxiv": null, "url": "https://icml.cc/Conferences/2009/papers/447.pdf", "date": "2009-01", "venue": { "name": "ICML 2009", "status": "conference" }, "authors": [ "Max Welling" ], "kind": "baseline", "settings": [ "image-classification" ], "families": [ "coreset-selection" ], "tier": "notable", "links": { "code": null, "project": null }, "status": { "verified": true, "verified_on": "2026-09-13", "source": "other", "depth": "metadata" }, "summary": "Defines herding, a deterministic weakly-chaotic dynamical system that converts a set of observed moments directly into a sequence of pseudo-samples respecting those moments, without first fitting a joint probability model over the data. Used in the coreset-selection literature as the mean-matching baseline: samples are added greedily to minimize the distance between the running average of selected samples' features and the true class-mean feature.", "abstract": null, "notes": "No arXiv version; verified against the ICML 2009 proceedings PDF (title, sole author). Abstract not reproduced (non-arXiv source)." } ] }