← Research

Decentralized Training

Training across independent participants and distributed compute.

2026

NeurIPS

AsyncMesh: Fully Asynchronous Optimization for Data and Pipeline Parallelism

T. Ajanthan, S. Ramasinghe, G. Avraham, H. Mohaghegh Dolatabadi, C. P. Hewa Koneputugodage, V. Shevchenko, Y. Zuo, A. Long

PDFarXivCode
BibTeX
@inproceedings{ajanthan_asyncmesh_arxiv26,
  title={{AsyncMesh: Fully Asynchronous Optimization for Data and Pipeline Parallelism}},
  author={Ajanthan, Thalaiyasingam and Ramasinghe, Sameera and Avraham, Gil and Mohaghegh Dolatabadi, Hadi and P Hewa Koneputugodage, Chamin and Shevchenko, Violetta and Zuo, Yan and Long, Alexander},
  year={2026},
  booktitle={NeurIPS}
}
NeurIPS

Communication-Efficient LLM Adaptation over Decentralized GPU Meshes

S. Ramasinghe, S. Siriwardhana, T. Ajanthan, H. Mohaghegh Dolatabadi, C. P. Hewa Koneputugodage, G. Avraham, V. Shevchenko, J. Snewin, K. Pajak, H. Xi, A. Long

PDFarXiv
BibTeX
@inproceedings{ramasinghe_llm_adaptation_arxiv26,
  title={{Communication-Efficient LLM Adaptation over Decentralized GPU Meshes}},
  author={Ramasinghe, Sameera and Siriwardhana, Shamane and Ajanthan, Thalaiyasingam and Mohaghegh Dolatabadi, Hadi and P Hewa Koneputugodage, Chamin and Avraham, Gil and Shevchenko, Violetta and Snewin, James and Pajak, Karol and Xi, Harry and Long, Alexander},
  year={2026},
  booktitle={NeurIPS}
}
ICML

Factored Gossip DiLoCo: Reducing Blocking Communication in DiLoCo

C. Hewa Koneputugodage, T. Ajanthan, S. Ramasinghe, H. Mohaghegh Dolatabadi, S. Siriwardhana, G. Avraham, V. Shevchenko, K. Pajak, J. Snewin, A. Long

PDFarXivPost
BibTeX
@inproceedings{koneputugodage_factored_gossip_icml26,
  title={{Factored Gossip DiLoCo: Reducing Blocking Communication in DiLoCo}},
  author={Hewa Koneputugodage, Chamin and Ajanthan, Thalaiyasingam and Ramasinghe, Sameera and Mohaghegh Dolatabadi, Hadi and Siriwardhana, Shamane and Avraham, Gil and Shevchenko, Violetta and Pajak, Karol and Snewin, James and Long, Alexander},
  year={2026},
  booktitle={ICML}
}
arXiv

Agora: Collective and Permissionless Internet-Scale Pretraining of Large Language Models

G. Avraham*, V. Shevchenko*, H. Mohaghegh Dolatabadi*, K. Pajak*, J. Snewin, H. Xi, R. O'Donnell, T. Ajanthan, S. Ramasinghe, C. Hewa Koneputugodage, S. Siriwardhana, A. Long

PDFarXivCode
BibTeX
@article{avraham_agora_arxiv26,
  title={{Agora: Collective and Permissionless Internet-Scale Pretraining of Large Language Models}},
  author={Avraham, Gil and Shevchenko, Violetta and Mohaghegh Dolatabadi, Hadi and Pajak, Karol and Snewin, James and Xi, Harry and O'Donnell, Rodney and Ajanthan, Thalaiyasingam and Ramasinghe, Sameera and Hewa Koneputugodage, Chamin and Siriwardhana, Shamane and Long, Alexander},
  year={2026},
  journal={arXiv},
  eprint={2607.13332},
  archiveprefix={arXiv},
  url={https://arxiv.org/abs/2607.13332}
}
arXiv

SENTINEL: Stagewise Integrity Verification for Pipeline Parallel Decentralized Training

H. Mohaghegh Dolatabadi, T. Ajanthan, S. Ramasinghe, C. P. Hewa Koneputugodage, G. Avraham, Y. Zuo, V. Shevchenko, A. Long

PDFarXiv
BibTeX
@article{dolatabadi_sentinel_arxiv26,
  title={{SENTINEL: Stagewise Integrity Verification for Pipeline Parallel Decentralized Training}},
  author={Mohaghegh Dolatabadi, Hadi and Ajanthan, Thalaiyasingam and Ramasinghe, Sameera and P Hewa Koneputugodage, Chamin and Avraham, Gil and Zuo, Yan and Shevchenko, Violetta and Long, Alexander},
  year={2026},
  journal={arXiv},
  eprint={2603.03592},
  archiveprefix={arXiv},
  url={https://arxiv.org/abs/2603.03592}
}

2025

NeurIPS

Mixtures of Subspaces for Bandwidth Efficient Context Parallel Training

S. Ramasinghe, T. Ajanthan, H. Mohaghegh Dolatabadi, G. Avraham, V. Shevchenko, Y. Zuo, C. P. Hewa Koneputugodage, A. Long

PDFarXiv
BibTeX
@inproceedings{ramasinghe_mixtures_neurips25,
  title={{Mixtures of Subspaces for Bandwidth Efficient Context Parallel Training}},
  author={Ramasinghe, Sameera and Ajanthan, Thalaiyasingam and Mohaghegh Dolatabadi, Hadi and Avraham, Gil and Shevchenko, Violetta and Zuo, Yan and P Hewa Koneputugodage, Chamin and Long, Alexander},
  year={2025},
  booktitle={NeurIPS}
}
NeurIPS

Subspace Networks: Scaling Decentralized Training with Communication-Efficient Model Parallelism

S. Ramasinghe, T. Ajanthan, G. Avraham, Y. Zuo, A. Long

PDFarXiv
BibTeX
@inproceedings{ramasinghe_protocol_neurips25,
  title={{Subspace Networks: Scaling Decentralized Training with Communication-Efficient Model Parallelism}},
  author={Ramasinghe, Sameera and Ajanthan, Thalaiyasingam and Avraham, Gil and Zuo, Yan and Long, Alexander},
  year={2025},
  booktitle={NeurIPS}
}
NeurIPS

Unextractable Protocol Models: Collaborative Training and Inference without Weight Materialization

A. Long*, C. P. Hewa Koneputugodage*, T. Ajanthan, Y. Zuo, G. Avraham, V. Shevchenko, H. Mohaghegh Dolatabadi, S. Ramasinghe

PDF
BibTeX
@inproceedings{long_upms_neurips25,
  title={{Unextractable Protocol Models: Collaborative Training and Inference without Weight Materialization}},
  author={Long, Alexander and P Hewa Koneputugodage, Chamin and Ajanthan, Thalaiyasingam and Zuo, Yan and Avraham, Gil and Shevchenko, Violetta and Mohaghegh Dolatabadi, Hadi and Ramasinghe, Sameera},
  year={2025},
  booktitle={NeurIPS}
}
ICLR Workshop

Exploring Asynchronism in SWARM Parallelism

Y. Zuo, G. Avraham, T. Ajanthan, S. Ramasinghe, A. Long

PDF
BibTeX
@inproceedings{zuo_asyncswarm_iclr25,
  title={{Exploring Asynchronism in SWARM Parallelism}},
  author={Zuo, Yan and Avraham, Gil and Ajanthan, Thalaiyasingam and Ramasinghe, Sameera and Long, Alexander},
  year={2025},
  booktitle={ICLR Workshop: Modularity for Collaborative, Decentralized, and Continual Deep Learning}
}
ICLR Workshop

Momentum Look-Ahead for Asynchronous Distributed Low-Communication Training

T. Ajanthan, S. Ramasinghe, G. Avraham, Y. Zuo, A. Long

PDF
BibTeX
@inproceedings{ajanthan_mla_iclr25,
  title={{Momentum Look-Ahead for Asynchronous Distributed Low-Communication Training}},
  author={Ajanthan, Thalaiyasingam and Ramasinghe, Sameera and Avraham, Gil and Zuo, Yan and Long, Alexander},
  year={2025},
  booktitle={ICLR Workshop: Modularity for Collaborative, Decentralized, and Continual Deep Learning}
}