% Merged bibliography for main.tex. Duplicate keys are kept from the earliest section file.

@book{BoydVandenberghe2004Convex,
  author    = {Stephen Boyd and Lieven Vandenberghe},
  title     = {Convex Optimization},
  publisher = {Cambridge University Press},
  year      = {2004},
  isbn      = {978-0-521-83378-3},
  url       = {https://web.stanford.edu/~boyd/cvxbook/}
}

@book{GoodfellowEtAl2016DeepLearning,
  author    = {Ian J. Goodfellow and Yoshua Bengio and Aaron C. Courville},
  title     = {Deep Learning},
  series    = {Adaptive Computation and Machine Learning},
  publisher = {MIT Press},
  year      = {2016},
  isbn      = {978-0-262-03561-3},
  url       = {https://www.deeplearningbook.org/}
}

@book{ManningEtAl2008IR,
  author    = {Christopher D. Manning and Prabhakar Raghavan and Hinrich Sch{"u}tze},
  title     = {Introduction to Information Retrieval},
  publisher = {Cambridge University Press},
  year      = {2008},
  isbn      = {978-0-521-86571-5},
  url       = {https://nlp.stanford.edu/IR-book/}
}

@book{Jolliffe2002PCA,
  author    = {Ian T. Jolliffe},
  title     = {Principal Component Analysis},
  edition   = {2},
  series    = {Springer Series in Statistics},
  publisher = {Springer},
  address   = {New York},
  year      = {2002},
  doi       = {10.1007/b98835},
  isbn      = {978-0-387-95442-4}
}

@book{SuttonBarto2018RL,
  author    = {Richard S. Sutton and Andrew G. Barto},
  title     = {Reinforcement Learning: An Introduction},
  edition   = {2},
  publisher = {MIT Press},
  year      = {2018},
  isbn      = {978-0-262-03924-6},
  url       = {http://incompleteideas.net/book/the-book-2nd.html}
}

% Neural networks, optimization, and regularization

@article{Rosenblatt1958Perceptron,
  author  = {Frank Rosenblatt},
  title   = {The Perceptron: A Probabilistic Model for Information Storage and Organization in the Brain},
  journal = {Psychological Review},
  volume  = {65},
  number  = {6},
  pages   = {386--408},
  year    = {1958},
  doi     = {10.1037/h0042519}
}

@article{RumelhartEtAl1986Backprop,
  author  = {David E. Rumelhart and Geoffrey E. Hinton and Ronald J. Williams},
  title   = {Learning Representations by Back-Propagating Errors},
  journal = {Nature},
  volume  = {323},
  number  = {6088},
  pages   = {533--536},
  year    = {1986},
  doi     = {10.1038/323533a0}
}

@inproceedings{SutskeverEtAl2013Momentum,
  author    = {Ilya Sutskever and James Martens and George E. Dahl and Geoffrey E. Hinton},
  title     = {On the Importance of Initialization and Momentum in Deep Learning},
  booktitle = {Proceedings of the 30th International Conference on Machine Learning},
  series    = {Proceedings of Machine Learning Research},
  volume    = {28},
  pages     = {1139--1147},
  publisher = {PMLR},
  year      = {2013},
  url       = {https://proceedings.mlr.press/v28/sutskever13.html}
}

@inproceedings{KingmaBa2015Adam,
  author    = {Diederik P. Kingma and Jimmy Ba},
  title     = {Adam: A Method for Stochastic Optimization},
  booktitle = {Proceedings of the 3rd International Conference on Learning Representations},
  year      = {2015},
  url       = {https://arxiv.org/abs/1412.6980}
}

@article{SrivastavaEtAl2014Dropout,
  author  = {Nitish Srivastava and Geoffrey E. Hinton and Alex Krizhevsky and Ilya Sutskever and Ruslan Salakhutdinov},
  title   = {Dropout: A Simple Way to Prevent Neural Networks from Overfitting},
  journal = {Journal of Machine Learning Research},
  volume  = {15},
  number  = {1},
  pages   = {1929--1958},
  year    = {2014},
  url     = {https://jmlr.org/papers/v15/srivastava14a.html}
}

@inproceedings{SzegedyEtAl2016LabelSmoothing,
  author    = {Christian Szegedy and Vincent Vanhoucke and Sergey Ioffe and Jonathon Shlens and Zbigniew Wojna},
  title     = {Rethinking the Inception Architecture for Computer Vision},
  booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
  pages     = {2818--2826},
  year      = {2016},
  doi       = {10.1109/CVPR.2016.308}
}

@inproceedings{HeEtAl2016ResNet,
  author    = {Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
  title     = {Deep Residual Learning for Image Recognition},
  booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
  pages     = {770--778},
  year      = {2016},
  doi       = {10.1109/CVPR.2016.90}
}

@article{BaEtAl2016LayerNorm,
  author        = {Lei Jimmy Ba and Jamie Ryan Kiros and Geoffrey E. Hinton},
  title         = {Layer Normalization},
  journal       = {arXiv preprint arXiv:1607.06450},
  year          = {2016},
  eprint        = {1607.06450},
  archiveprefix = {arXiv},
  url           = {https://arxiv.org/abs/1607.06450}
}

% Evaluation and representation learning

@inproceedings{DrorEtAl2018Significance,
  author    = {Rotem Dror and Gili Baumer and Segev Shlomov and Roi Reichart},
  title     = {The Hitchhiker's Guide to Testing Statistical Significance in Natural Language Processing},
  booktitle = {Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics},
  pages     = {1383--1392},
  publisher = {Association for Computational Linguistics},
  year      = {2018},
  doi       = {10.18653/v1/P18-1128}
}

@article{BengioEtAl2013Representation,
  author  = {Yoshua Bengio and Aaron C. Courville and Pascal Vincent},
  title   = {Representation Learning: A Review and New Perspectives},
  journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
  volume  = {35},
  number  = {8},
  pages   = {1798--1828},
  year    = {2013},
  doi     = {10.1109/TPAMI.2013.50}
}

@article{BengioEtAl2003NeuralLM,
  author  = {Yoshua Bengio and R{\'e}jean Ducharme and Pascal Vincent and Christian Janvin},
  title   = {A Neural Probabilistic Language Model},
  journal = {Journal of Machine Learning Research},
  volume  = {3},
  pages   = {1137--1155},
  year    = {2003},
  url     = {https://jmlr.org/papers/v3/bengio03a.html}
}

@article{OordEtAl2018CPC,
  author        = {A{"a}ron van den Oord and Yazhe Li and Oriol Vinyals},
  title         = {Representation Learning with Contrastive Predictive Coding},
  journal       = {arXiv preprint arXiv:1807.03748},
  year          = {2018},
  eprint        = {1807.03748},
  archiveprefix = {arXiv},
  url           = {https://arxiv.org/abs/1807.03748}
}

@inproceedings{ChenEtAl2020SimCLR,
  author    = {Ting Chen and Simon Kornblith and Mohammad Norouzi and Geoffrey E. Hinton},
  title     = {A Simple Framework for Contrastive Learning of Visual Representations},
  booktitle = {Proceedings of the 37th International Conference on Machine Learning},
  series    = {Proceedings of Machine Learning Research},
  volume    = {119},
  pages     = {1597--1607},
  publisher = {PMLR},
  year      = {2020},
  url       = {https://proceedings.mlr.press/v119/chen20j.html}
}

@inproceedings{MikolovEtAl2013Word2Vec,
  author    = {Tom{\'a}s Mikolov and Kai Chen and Greg Corrado and Jeffrey Dean},
  title     = {Efficient Estimation of Word Representations in Vector Space},
  booktitle = {Proceedings of the 1st International Conference on Learning Representations, Workshop Track},
  year      = {2013},
  url       = {https://arxiv.org/abs/1301.3781}
}

@inproceedings{PenningtonEtAl2014GloVe,
  author    = {Jeffrey Pennington and Richard Socher and Christopher D. Manning},
  title     = {{GloVe}: Global Vectors for Word Representation},
  booktitle = {Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing},
  pages     = {1532--1543},
  publisher = {Association for Computational Linguistics},
  year      = {2014},
  doi       = {10.3115/v1/D14-1162}
}

@inproceedings{PapineniEtAl2002BLEU,
  author    = {Kishore Papineni and Salim Roukos and Todd Ward and Wei{-}Jing Zhu},
  title     = {{BLEU}: A Method for Automatic Evaluation of Machine Translation},
  booktitle = {Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics},
  pages     = {311--318},
  publisher = {Association for Computational Linguistics},
  year      = {2002},
  doi       = {10.3115/1073083.1073135}
}

@article{PanYang2010Transfer,
  author  = {Sinno Jialin Pan and Qiang Yang},
  title   = {A Survey on Transfer Learning},
  journal = {IEEE Transactions on Knowledge and Data Engineering},
  volume  = {22},
  number  = {10},
  pages   = {1345--1359},
  year    = {2010},
  doi     = {10.1109/TKDE.2009.191}
}

% Sequence models and Transformers

@inproceedings{SutskeverEtAl2014Seq2Seq,
  author    = {Ilya Sutskever and Oriol Vinyals and Quoc V. Le},
  title     = {Sequence to Sequence Learning with Neural Networks},
  booktitle = {Advances in Neural Information Processing Systems 27},
  pages     = {3104--3112},
  year      = {2014},
  url       = {https://proceedings.neurips.cc/paper/2014/hash/a14ac55a4f27472c5d894ec1c3c743d2-Abstract.html}
}

@inproceedings{BahdanauEtAl2015Attention,
  author    = {Dzmitry Bahdanau and Kyunghyun Cho and Yoshua Bengio},
  title     = {Neural Machine Translation by Jointly Learning to Align and Translate},
  booktitle = {Proceedings of the 3rd International Conference on Learning Representations},
  year      = {2015},
  url       = {https://arxiv.org/abs/1409.0473}
}

@inproceedings{VaswaniEtAl2017Transformer,
  author    = {Ashish Vaswani and Noam Shazeer and Niki Parmar and Jakob Uszkoreit and Llion Jones and Aidan N. Gomez and Lukasz Kaiser and Illia Polosukhin},
  title     = {Attention Is All You Need},
  booktitle = {Advances in Neural Information Processing Systems 30},
  pages     = {5998--6008},
  year      = {2017},
  url       = {https://proceedings.neurips.cc/paper/2017/hash/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html}
}

% Pre-training and transfer across modalities

@inproceedings{HeEtAl2022MAE,
  author    = {Kaiming He and Xinlei Chen and Saining Xie and Yanghao Li and Piotr Doll{\'a}r and Ross B. Girshick},
  title     = {Masked Autoencoders Are Scalable Vision Learners},
  booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages     = {15979--15988},
  year      = {2022},
  doi       = {10.1109/CVPR52688.2022.01553}
}

@inproceedings{BaevskiEtAl2020Wav2vec,
  author    = {Alexei Baevski and Yuhao Zhou and Abdelrahman Mohamed and Michael Auli},
  title     = {wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations},
  booktitle = {Advances in Neural Information Processing Systems 33},
  year      = {2020},
  url       = {https://proceedings.neurips.cc/paper/2020/hash/92d1e1eb1cd6f9fba3227870bb6d7f07-Abstract.html}
}

@inproceedings{DevlinEtAl2019BERT,
  author    = {Jacob Devlin and Ming{-}Wei Chang and Kenton Lee and Kristina Toutanova},
  title     = {{BERT}: Pre-training of Deep Bidirectional Transformers for Language Understanding},
  booktitle = {Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
  pages     = {4171--4186},
  publisher = {Association for Computational Linguistics},
  year      = {2019},
  doi       = {10.18653/v1/N19-1423}
}

@inproceedings{LewisEtAl2020BART,
  author    = {Mike Lewis and Yinhan Liu and Naman Goyal and Marjan Ghazvininejad and Abdelrahman Mohamed and Omer Levy and Veselin Stoyanov and Luke Zettlemoyer},
  title     = {{BART}: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension},
  booktitle = {Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
  pages     = {7871--7880},
  publisher = {Association for Computational Linguistics},
  year      = {2020},
  doi       = {10.18653/v1/2020.acl-main.703}
}

@article{RaffelEtAl2020T5,
  author  = {Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu},
  title   = {Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer},
  journal = {Journal of Machine Learning Research},
  volume  = {21},
  number  = {140},
  pages   = {1--67},
  year    = {2020},
  url     = {https://jmlr.org/papers/v21/20-074.html}
}

@inproceedings{BrownEtAl2020GPT3,
  author    = {Tom B. Brown and Benjamin Mann and Nick Ryder and Melanie Subbiah and Jared Kaplan and Prafulla Dhariwal and Arvind Neelakantan and Pranav Shyam and Girish Sastry and Amanda Askell and Sandhini Agarwal and Ariel Herbert{-}Voss and Gretchen Krueger and Tom Henighan and Rewon Child and Aditya Ramesh and Daniel M. Ziegler and Jeffrey Wu and Clemens Winter and Christopher Hesse and Mark Chen and Eric Sigler and Mateusz Litwin and Scott Gray and Benjamin Chess and Jack Clark and Christopher Berner and Sam McCandlish and Alec Radford and Ilya Sutskever and Dario Amodei},
  title     = {Language Models Are Few-Shot Learners},
  booktitle = {Advances in Neural Information Processing Systems 33},
  pages     = {1877--1901},
  year      = {2020},
  url       = {https://proceedings.neurips.cc/paper/2020/hash/1457c0d6bfcb4967418bfb8ac142f64a-Abstract.html}
}

% Entries transferred from the reference textbook for overlapping material

@article{Chen-and-Goodman:1999,
  title   = {An Empirical Study of Smoothing Techniques for Language Modeling},
  author  = {Stanley F. Chen and Joshua Goodman},
  journal = {Computer Speech and Language},
  volume  = {13},
  pages   = {359--394},
  year    = {1999}
}

@article{Cortes-and-Vapnik:1995,
  title   = {Support-Vector Networks},
  author  = {Corinna Cortes and Vladimir Vapnik},
  journal = {Machine Learning},
  volume  = {20},
  pages   = {273--297},
  year    = {1995}
}

@book{Mitchell:1997,
  title     = {Machine Learning},
  author    = {Tom M. Mitchell},
  publisher = {McGraw-Hill Education},
  year      = {1997}
}

@inproceedings{Collobert-and-Weston:2008AUA,
  title     = {A Unified Architecture for Natural Language Processing: Deep Neural Networks with Multitask Learning},
  author    = {Ronan Collobert and Jason Weston},
  booktitle = {Proceedings of the 25th International Conference on Machine Learning},
  pages     = {160--167},
  year      = {2008}
}

@book{Hastie-etal:2009,
  title     = {The Elements of Statistical Learning},
  author    = {Trevor Hastie and Robert Tibshirani and Jerome Friedman},
  publisher = {Springer},
  year      = {2009}
}

@book{McClave-and-Sincich-2006,
  title     = {Statistics},
  edition   = {10},
  author    = {James T. McClave and Terry Sincich},
  publisher = {Prentice Hall},
  year      = {2006}
}

@book{Freedman-et-al-2007,
  title     = {Statistics},
  edition   = {4},
  author    = {David Freedman and Robert Pisani and Roger Purves},
  publisher = {W. W. Norton \& Company},
  year      = {2007}
}

@book{Freedman:2009,
  title     = {Statistical Models: Theory and Practice},
  edition   = {2},
  author    = {David A. Freedman},
  publisher = {Cambridge University Press},
  year      = {2009}
}

@book{Dror-et-al:2020,
  title     = {Neural Network Methods for Natural Language Processing},
  author    = {Rotem Dror and Lotem Peled-Cohen and Segev Shlomov},
  publisher = {Morgan \& Claypool Publishers},
  year      = {2020}
}

@article{hinton-etal:2012improving,
  title   = {Improving Neural Networks by Preventing Co-adaptation of Feature Detectors},
  author  = {Geoffrey E. Hinton and Nitish Srivastava and Alex Krizhevsky and Ilya Sutskever and Ruslan R. Salakhutdinov},
  journal = {arXiv preprint arXiv:1207.0580},
  year    = {2012}
}

@incollection{prechelt1998early,
  title     = {Early Stopping---But When?},
  author    = {Lutz Prechelt},
  booktitle = {Neural Networks: Tricks of the Trade},
  pages     = {55--69},
  publisher = {Springer},
  year      = {1998}
}

@techreport{plaut-etal:1986experiments,
  title       = {Experiments on Learning by Back Propagation},
  author      = {David C. Plaut and Steven J. Nowlan and Geoffrey E. Hinton},
  institution = {Carnegie-Mellon University},
  year        = {1986}
}

@article{Holmstrom-Koistinen:1992noise,
  title   = {Using Additive Noise in Back-Propagation Training},
  author  = {Lasse Holmstr{\"o}m and Petri Koistinen},
  journal = {IEEE Transactions on Neural Networks},
  volume  = {3},
  number  = {1},
  pages   = {24--38},
  year    = {1992}
}

@article{bishop:1995training,
  title   = {Training with Noise Is Equivalent to {Tikhonov} Regularization},
  author  = {Christopher M. Bishop},
  journal = {Neural Computation},
  volume  = {7},
  number  = {1},
  pages   = {108--116},
  year    = {1995}
}

@inproceedings{graves-etal:2013speech,
  title     = {Speech Recognition with Deep Recurrent Neural Networks},
  author    = {Alex Graves and Abdel-rahman Mohamed and Geoffrey E. Hinton},
  booktitle = {2013 IEEE International Conference on Acoustics, Speech and Signal Processing},
  pages     = {6645--6649},
  year      = {2013}
}

@article{neelakantan-etal:2015adding,
  title   = {Adding Gradient Noise Improves Learning for Very Deep Networks},
  author  = {Arvind Neelakantan and Luke Vilnis and Quoc V. Le and Ilya Sutskever and Lukasz Kaiser and Karol Kurach and James Martens},
  journal = {arXiv preprint arXiv:1511.06807},
  year    = {2015}
}

@inproceedings{sennrich-etal:2016improving,
  title     = {Improving Neural Machine Translation Models with Monolingual Data},
  author    = {Rico Sennrich and Barry Haddow and Alexandra Birch},
  booktitle = {Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics},
  pages     = {86--96},
  year      = {2016}
}

@inproceedings{feng-etal:2021survey,
  title     = {A Survey of Data Augmentation Approaches for {NLP}},
  author    = {Steven Y. Feng and Varun Gangal and Jason Wei and Sarath Chandar and Soroush Vosoughi and Teruko Mitamura and Eduard Hovy},
  booktitle = {Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021},
  pages     = {968--988},
  year      = {2021}
}

@article{shorten-Khoshgoftaar:2019survey,
  title   = {A Survey on Image Data Augmentation for Deep Learning},
  author  = {Connor Shorten and Taghi M. Khoshgoftaar},
  journal = {Journal of Big Data},
  volume  = {6},
  number  = {1},
  pages   = {1--48},
  year    = {2019}
}

@inproceedings{Szegedy-etal:2014neural,
  title     = {Intriguing Properties of Neural Networks},
  author    = {Christian Szegedy and Wojciech Zaremba and Ilya Sutskever and Joan Bruna and Dumitru Erhan and Ian Goodfellow and Rob Fergus},
  booktitle = {Proceedings of the 2nd International Conference on Learning Representations},
  year      = {2014}
}

@inproceedings{Goodfellow-etal:2015Adversarial,
  title     = {Explaining and Harnessing Adversarial Examples},
  author    = {Ian Goodfellow and Jonathon Shlens and Christian Szegedy},
  booktitle = {Proceedings of the 3rd International Conference on Learning Representations},
  year      = {2015}
}

@article{duchi-etal:2011adaptive,
  title   = {Adaptive Subgradient Methods for Online Learning and Stochastic Optimization},
  author  = {John Duchi and Elad Hazan and Yoram Singer},
  journal = {Journal of Machine Learning Research},
  volume  = {12},
  pages   = {2121--2159},
  year    = {2011}
}

@misc{Hinton:2018RMSProp,
  author = {Geoffrey E. Hinton},
  title  = {Neural Networks for Machine Learning, Lecture 6},
  url    = {http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf},
  year   = {2018}
}

@article{bengio-etal:2000neural,
  title   = {A Neural Probabilistic Language Model},
  author  = {Yoshua Bengio and R{\'e}jean Ducharme and Pascal Vincent},
  journal = {Advances in Neural Information Processing Systems},
  volume  = {13},
  year    = {2000}
}

@book{Kelly-Stone:1975senses,
  title     = {Computer Recognition of English Word Senses},
  author    = {Edward F. Kelly and Philip J. Stone},
  publisher = {American Elsevier Pub},
  year      = {1975}
}

@article{harris:1954distributional,
  title   = {Distributional Structure},
  author  = {Zellig S. Harris},
  journal = {Word},
  volume  = {10},
  number  = {2--3},
  pages   = {146--162},
  year    = {1954}
}

@article{firth:1957synopsis,
  title   = {A Synopsis of Linguistic Theory, 1930--1955},
  author  = {John R. Firth},
  journal = {Studies in Linguistic Analysis},
  year    = {1957}
}

@inproceedings{cho-etal:2014learning,
  title     = {Learning Phrase Representations Using {RNN} Encoder--Decoder for Statistical Machine Translation},
  author    = {Kyunghyun Cho and Bart van Merri{\"e}nboer and Caglar Gulcehre and Dzmitry Bahdanau and Fethi Bougares and Holger Schwenk and Yoshua Bengio},
  booktitle = {Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing},
  pages     = {1724--1734},
  year      = {2014}
}

@article{graves-etal:2014neural,
  title   = {Neural {Turing} Machines},
  author  = {Alex Graves and Greg Wayne and Ivo Danihelka},
  journal = {arXiv preprint arXiv:1410.5401},
  year    = {2014}
}

@inproceedings{luong-etal:2015effective,
  title     = {Effective Approaches to Attention-Based Neural Machine Translation},
  author    = {Minh-Thang Luong and Hieu Pham and Christopher D. Manning},
  booktitle = {Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing},
  pages     = {1412--1421},
  year      = {2015}
}

@article{chaudhari-etal:2021attentive,
  title   = {An Attentive Survey of Attention Models},
  author  = {Sneha Chaudhari and Varun Mithal and Gungor Polatkan and Rohan Ramanath},
  journal = {ACM Transactions on Intelligent Systems and Technology},
  volume  = {12},
  number  = {5},
  pages   = {1--32},
  year    = {2021}
}

@article{radford-etal:2018improving,
  title     = {Improving Language Understanding by Generative Pre-Training},
  author    = {Alec Radford and Karthik Narasimhan and Tim Salimans and Ilya Sutskever},
  journal   = {OpenAI Technical Report},
  publisher = {OpenAI},
  year      = {2018}
}

@article{bengio-etal:2006greedy,
  title   = {Greedy Layer-Wise Training of Deep Networks},
  author  = {Yoshua Bengio and Pascal Lamblin and Dan Popovici and Hugo Larochelle},
  journal = {Advances in Neural Information Processing Systems},
  volume  = {19},
  year    = {2006}
}

@inproceedings{erhan-etal:2010does,
  title     = {Why Does Unsupervised Pre-Training Help Deep Learning?},
  author    = {Dumitru Erhan and Aaron Courville and Yoshua Bengio and Pascal Vincent},
  booktitle = {Proceedings of the Thirteenth International Conference on Artificial Intelligence and Statistics},
  pages     = {201--208},
  year      = {2010}
}

@book{Hemingway:2002,
    title={Old Man and the Sea},
    author={Ernest Hemingway},
    publisher={Scribner},
    year={2002}
}

@book{Chomsky:1973,
    title={For Reasons of State (1st ed.)},
    author={Noam Chomsky},
    publisher={Pantheon},
    year={1973}
}

@book{Chomsky:1975,
    title={Reflections on Language},
    author={Noam Chomsky},
    publisher={Pantheon},
    year={1975}
}

@book{Amberg-and-Vause:2009,
    title={American English: History, Structure, and Usage},
    author={Julie S. Amberg and Deborah J. Vause},
    publisher={Cambridge University Press},
    year={2009}
}

@book{Russell-and-Norvig:2010,
    title={Artificial Intelligence: A Modern Approach (3nd ed.)},
    author={Stuart J. Russell and Peter Norvig},
    publisher={Prentice Hall},
    year={2010}
}

@book{Booij:2007,
    title={The Grammar of Words: An Introduction to Linguistic Morphology (2nd ed.)},
    author={Geert Booij},
    publisher={Oxford University Press},
    year={2007}
}

@book{Altshuler-etal:2019,
    title={A Course in Semantics},
    author={Daniel Altshuler and Terence Parsons and Roger Schwarzschild},
    publisher={The MIT Press},
    year={2019}
}

@article{Turing:1950,
    title={Computing machinery and intelligence},
    author={Alan Turing},
    journal={Mind},
    volume={59(236)},
    pages={433-450},
    year={1950}
}

@article{hassanawadalla2018achieving,
    author={Hany Hassan and Anthony Aue and Chang Chen and Vishal Chowdhary and Jonathan Clark and Christian Federmann and Xuedong Huang and Marcin Junczys{-}Dowmunt and William Lewis and Mu Li and Shujie Liu and Tie{-}Yan Liu and Renqian Luo and Arul Menezes and Tao Qin and Frank Seide and Xu Tan and Fei Tian and Lijun Wu and Shuangzhi Wu and Yingce Xia and Dongdong Zhang and Zhirui Zhang and Ming Zhou},
    title={Achieving Human Parity on Automatic Chinese to English News Translation},
    journal={CoRR},
    year={2018},
    eprinttype={arXiv},
    eprint={1803.05567},
}

@inproceedings{barrault-etal:2019findings,
    title={Findings of the 2019 Conference on Machine Translation (WMT19)},
    author={Hany Hassan and Anthony Aue and Chang Chen and Vishal Chowdhary and Jonathan Clark and Christian Federmann and Xuedong Huang and Marcin Junczys-Dowmunt and William Lewis and Mu Li and Shujie Liu and Tie-Yan Liu and Renqian Luo and Arul Menezes and Tao Qin and Frank Seide and Xu Tan and Fei Tian and Lijun Wu and Shuangzhi Wu and Yingce Xia and Dongdong Zhang and Zhirui Zhang and Ming Zhou},
    booktitle={Proceedings of the Fourth Conference on Machine Translation (Volume 2: Shared Task Papers)},
    pages={1--61},
    year={2019}
}

@book{Jurafsky-and-Martin:2008,
    title={Speech and Language Processing (2nd ed.)},
    author={Dan Jurafsky and James H. Martin},
    publisher={Prentice Hall},
    year={2008}
}

@book{Dry-and-Lawler:1998,
    title={Using Computers in Linguistics: A Practical Guide},
    author={Helen Aristar Dry and John Lawler},
    publisher={Routledge},
    year={1998}
}

@book{Pearl-and-Mackenzie:2018,
    title={The Book of Why: The New Science of Cause and Effect},
    author={Judea Pearl and Dana Mackenzie},
    publisher={Allen Lane},
    year={2018}
}

@book{Empson:1966,
    title={Seven Types of Ambiguity},
    author={William Empson},
    publisher={New Directions},
    year={1966}
}

@article{Abler:2006,
    author={William L. Abler},
    title={The nature of Language},
    journal={Science Progress},
    volume={89},
    number={1},
    pages={61-70},
    year={2006}
}

@misc{www.ethnologue.com,
    howpublished={\url{http://www.ethnologue.com}},
    note={Retrieved 2021-02-26},
    title={Ethnologue: Languages of the World. Twenty-fourth edition},
    author={David M. Eberhard and Gary F. Simons and Charles D. Fennig},
    year={2021}
}

@book{Frege:1884,
    title={Die Grundlagen der Arithmetik. Eine logisch-mathematische Untersuchung {\"u}ber den Begriff der Zahl},
    author={Gottlob Frege},
    publisher={Breslau: Wilhelm Koebner},
    year={1884}
}

@book{Frege:Austin:1980,
    title={The Foundations of Arithmetic. A Logico-Mathematical Enquiry into the Concept of Number},
    author={Gottlob Frege},
    translator={J. L. Austin},
    publisher={Northwestern University Press},
    year={1980}
}

@incollection{Janssen:2012,
    title={Compositionality: its historic context},
    author={Theo M.V. Janssen},
    booktitle={The Oxford handbook of compositionality},
    editor={M. Werning and W. Hinzen and E. Machery},
    publisher={Oxford University Press},
    year={2012}
}

@book{Whitehead-and-Price:2008,
    title={Dialogues of Alfred North Whitehead (2nd ed.)},
    author={Alfred North Whitehead and Lucien Price},
    publisher={David R. Godine Publisher},
    year={2008}
}

@book{Manning-and-Schutze:1999,
    title={Foundations of Statistical Natural Language Processing},
    author={Chris Manning and Hinrich Sch{\"u}tze},
    publisher={The MIT Press},
    year={1999}
}

@book{Hirst:2021,
    title={Synthesis Lectures on Human Language Technologies},
    author={Graeme Hirst},
    publisher={Morgan {\&} Claypool Publishers},
    year={2021}
}

@book{Mitkov:2014,
    title={The Oxford Handbook of Computational Linguistics (2nd ed.) },
    author={Ruslan Mitkov},
    publisher={Oxford University Press},
    year={2014}
}

@book{Clark-etal:2012,
    title={The Handbook of Computational Linguistics and Natural Language Processing},
    author={Alexander Clark and Chris Fox and Shalom Lappin},
    publisher={Wiley-Blackwell},
    year={2012}
}

@book{Lane-etal:2019,
    title={Natural Language Processing in Action},
    author={Hobson Lane and Hannes Hapke and Cole Howard},
    publisher={Manning Publications},
    year={2019}
}

@book{Eisenstein:2019,
    title={Introduction to Natural Language Processing},
    author={Jacob Eisenstein},
    publisher={The MIT Press},
    year={2019}
}

@book{Zong-etal:2021,
    title={Text Data Mining},
    author={Chengqing Zong and Rui Xia and Jiajun Zhang},
    publisher={Springer},
    year={2021}
}

@book{Koehn:2010,
    title={Statistical Machine Translation},
    author={Philipp Koehn},
    publisher={Cambridge University Press},
    year={2010}
}

@book{Koehn:2020,
    title={Neural Machine Translation},
    author={Philipp Koehn},
    publisher={Cambridge University Press},
    year={2020}
}

@book{Manning-etal:2008,
    title={Introduction to Information Retrieval},
    author={Christopher D. Manning and Prabhakar Raghavan and Hinrich Sch{\"u}tze},
    publisher={Cambridge University Press},
    year={2008}
}

@book{Liu:2011,
    title={Web Data Mining (2nd ed.)},
    author={Bing Liu},
    publisher={Springer},
    year={2011}
}

@book{Jelinek:1998,
    title={Statistical Methods for Speech Recognition (4th ed.)},
    author={Frederick Jelinek},
    publisher={A Bradford Book},
    year={1998}
}

@book{Goldberg:2017,
    title={Neural Network Methods for Natural Language Processing},
    author={Yoav Goldberg},
    publisher={Morgan {\&} Claypool Publishers},
    year={2017}
}

@book{Deng-and-Liu:2018,
    title={Deep Learning in Natural Language Processing},
    author={Li Deng and Yang Liu},
    publisher={Springer},
    year={2018}
}

@book{Zhang-and-Teng:2021,
    title={Natural Language Processing: A Machine Learning Perspective},
    author={Yue Zhang and Zhiyang Teng},
    publisher={Cambridge University Press},
    year={2021}
}

@book{Chomsky:2006,
    title={Language and Mind (3rd ed.)},
    author={Noam Chomsky},
    publisher={Cambridge University Press},
    year={2006}
}

@book{Fromkin-etal:2013,
    title={An Introduction to Language (10th ed.)},
    author={Victoria Fromkin and Robert Rodman and Nina Hyams},
    publisher={Cengage Learning},
    year={2013}
}

@book{Pinker:1994,
    title={The Language Instinct},
    author={Steven Pinker},
    publisher={William Morrow and Company},
    year={1994}
}

@book{Chomsky:2017,
    title={On Language: Language and Responsibility and Reflections on Language},
    author={Noam Chomsky},
    publisher={The New Press},
    year={2017}
}

@book{Deacon:1998,
    title={The Symbolic Species: The Co-evolution of Language and the Brain},
    author={Terry Deacon},
    publisher={W. W. Norton {\&} Company},
    year={1998}
}

@book{Bender:2013,
    title={Linguistic Fundamentals for Natural Language Processing: 100 Essentials from Morphology and Syntax},
    author={Emily M. Bender},
    publisher={Morgan {\&} Claypool Publishers},
    year={2013}
}

@book{Bender-and-Lascarides:2019,
    title={Linguistic Fundamentals for Natural Language Processing II: 100 Essentials from Semantics and Pragmatics},
    author={Emily M. Bender and Alex Lascarides},
    publisher={Morgan {\&} Claypool Publishers},
    year={2019}
}

@article{wei-etal:2022emergent,
    title={Emergent abilities of large language models},
    author={Jason Wei and Yi Tay and Rishi Bommasani and Colin Raffel and Barret Zoph and Sebastian Borgeaud and Dani Yogatama and Maarten Bosma and Denny Zhou and Donald Metzler and Ed H. Chi and Tatsunori Hashimoto and Oriol Vinyals and Percy Liang and Jeff Dean and William Fedus},
    journal={arXiv preprint arXiv:2206.07682},
    year={2022}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CHAPTER 2

@book{Murphy:2012,
    title={Machine Learning: A Probabilistic Perspective},
    author={Kevin P. Murphy},
    publisher={MIT Press},
    year={2012}
}

@book{Mohri-et-al:2018,
    title={Foundations of Machine Learning (2nd ed.)},
    author={Mehryar Mohri and Afshin Rostamizadeh and Ameet Talwalkar},
    publisher={MIT Press},
    year={2018}
}

@book{Ash-et-al:1999,
    title={Probability {\&} Measure Theory},
    author={Robert B. Ash and Catherine A. Dol\'eans-Dade},
    publisher={Academic Press},
    year={1999}
}

@techreport{Shannon:1948,
     author = {Claude E. Shannon},
     title = {A Mathematical Theory of Communication},
     type = {Report},
     publisher = {Bell System Technical Journal},
     institution = {Bell Labs},
     year = {1948}
}

@book{Bishop:2006,
    title={Pattern Recognition and Machine Learning},
    author={Christopher M. Bishop},
    publisher={Springer},
    year={2006}
}

@book{Hurley:2011,
    title={A Concise Introduction to Logic (11th ed.)},
    author={Patrick Hurley},
    publisher={Wadsworth Publishing},
    year={2011}
}

@article{Bengio-et-al:2003,
    title={A Neural Probabilistic Language Model},
    author={Yoshua Bengio and R\'ejean Ducharme and Pascal Vincent and Christian Jauvin},
    journal={Journal of Machine Learning Research},
    volume={3},
    pages={1137-1155},
    year={2003}
}

@inproceedings{Och-and-Ney:2002,
    title = "Discriminative Training and Maximum Entropy Models for Statistical Machine Translation",
    author = "Och, Franz Josef  and Ney, Hermann",
    booktitle = "Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics",
    year = "2002",
    pages = "295-302",
}

@article{Wolpert:1996,
    title={The Lack of A Priori Distinctions Between Learning Algorithms},
    author={David H. Wolpert},
    journal={Neural Computatoin},
    volume={8(7)},
    pages={1341-1390},
    year={1996}
}

@article{Wolpert-and-Macready:1997,
    title={No Free Lunch Theorems for Optimization},
    author={David H. Wolpert and William G. Macready},
    journal={IEEE Transactions on Evolutionary Computation},
    volume={1(1)},
    pages={67-82},
    year={1997}
}

@article{Vapnik-and-Chervonenkis:1971,
    title={On the Uniform Convergence of Relative Frequencies of Events to Their Probabilities},
    author={Vladimir Vapnik and Alexey Chervonenkis},
    journal={Theory of Probability {\&} Its Applications},
    volume={16(2)},
    pages={264-279},
    year={1971}
}

@book{Konishi-and-Kitagawa:2007,
    title={Information Criteria and Statistical Modeling},
    author={Sadanori Konishi and Genshiro Kitagawa},
    publisher={Spinger},
    year={2007}
}

@book{Burnham-and-Anderson:2002,
    title={Model selection and multimodel inference: a practical information-theoretic approach},
    author={Kenneth P. Burnham and David R. Anderson},
    publisher={Spinger},
    year={2002}
}

@book{Goodfellow-et-al-2016,
    title={Deep Learning},
    author={Ian Goodfellow and Yoshua Bengio and Aaron Courville},
    publisher={MIT Press},
    year={2016}
}

@book{Seni-et-al-2010,
    title={Ensemble Methods in Data Mining: Improving Accuracy Through Combining Predictions},
    author={Giovanni Seni and John Elder and Robert Grossman},
    publisher={Morgan and Claypool Publishers},
    year={2010}
}

@book{Zhou-2012,
    title={Ensemble Methods: Foundations and Algorithms},
    author={Zhi-Hua Zhou},
    publisher={Chapman and Hall/CRC},
    year={2012}
}

@article{breiman96,
    author = {Leo  Breiman},
    journal = {Machine Learning},
    number = {2},
    pages = {123-140},
    title = {Bagging Predictors},
    volume = {24},
    year = {1996}
}

@article{schapire-1990strength,
    author = {Robert E. Schapire},
    journal = {Machine Learning},
    number = {2},
    pages = {197--227},
    title = {The strength of weak learnability},
    volume = {5},
    year = {1990}
}

@inproceedings{Papineni-et-al2002bleu:a,
    author = {Kishore Papineni and Salim Roukos and Todd Ward and Wei-jing Zhu},
    title = {BLEU: a Method for Automatic Evaluation of Machine Translation},
    booktitle = {Proceedings of the 40th Annual meeting of the Association for Computational Linguistics},
    year = {2002},
    pages = {311--318}
}

@inproceedings{Lafferty-et-al:2001crf,
    author = {John Lafferty and Andrew McCallum and Fernando Pereira},
    title = {Conditional random fields: Probabilistic models for segmenting and labeling sequence data},
    booktitle = {Proceedings of the 18th International Conference on Machine Learning 2001},
    year = {2001},
    pages = {282--289}
}

@inproceedings{ramshaw-marcus:1995text,
    title = "Text Chunking using Transformation-Based Learning",
    author = "Ramshaw, Lance  and Marcus, Mitch",
    booktitle = "Third Workshop on Very Large Corpora",
    year = "1995"
}

@inproceedings{yarowsky:1994decision,
    title = "DECISION LISTS FOR LEXICAL AMBIGUITY RESOLUTION: Application to Accent Restoration in {S}panish and {F}rench",
    author = "Yarowsky, David",
    booktitle = "Proceedings of the 32nd Annual Meeting of the Association for Computational Linguistics",
    year = "1994",
    pages = "88--95"
}

@inproceedings{pang-etal:2002thumbs,
    title = "Thumbs up? Sentiment Classification using Machine Learning Techniques",
    author = "Pang, Bo  and Lee, Lillian  and Vaithyanathan, Shivakumar",
    booktitle = "Proceedings of the 2002 Conference on Empirical Methods in Natural Language Processing ({EMNLP} 2002)",
    year = "2002",
    pages = "79--86"
}

@inproceedings{krebs-et-al:2018semeval,
    title = "{S}em{E}val-2018 Task 10: Capturing Discriminative Attributes",
    author = "Krebs, Alicia and Lenci, Alessandro and Paperno, Denis",
    booktitle = "Proceedings of The 12th International Workshop on Semantic Evaluation",
    year = "2018",
    pages = "732--740"
}

@article{gildea-jurafsky:2002automatic,
    title = "Automatic Labeling of Semantic Roles",
    author = "Gildea, Daniel and Jurafsky, Daniel",
    journal = "Computational Linguistics",
    volume = "28",
    number = "3",
    year = "2002",
    pages = "245--288"
}

@inproceedings{Bahl-and-Mercer:1976,
  author = {Bahl, L. R. and Mercer, R. L.},
  booktitle = {Proceedings of IEEE International Symposium on Information Theory},
  pages = {88--89},
  title = {Part of Speech Assignment by a Statistical Decision Algorithm},
  year = {1976}
}

@inproceedings{tjong-kim-sang-buchholz:2000introduction,
    title = "Introduction to the {C}o{NLL}-2000 Shared Task Chunking",
    author = "Tjong Kim Sang, Erik F.  and Buchholz, Sabine",
    booktitle = "Proceedings of Fourth Conference on Computational Natural Language Learning and the Second Learning Language in Logic Workshop",
    year = "2000"
}

@inproceedings{tjong-kim-sang:2002introduction,
    title = "Introduction to the {C}o{NLL}-2002 Shared Task: Language-Independent Named Entity Recognition",
    author = "Tjong Kim Sang, Erik F.",
    booktitle = "Proceedings of {COLING}-02: The 6th Conference on Natural Language Learning 2002 ({C}o{NLL}-2002)",
    year = "2002"
}

@incollection{Fellbaum:2005,
    title={WordNet and wordnets},
    author={Christiane Fellbaum},
    booktitle={Encyclopedia of Language and Linguistics (2nd ed.)},
    editor={Keith Brown},
    publisher={Elsevier},
    year={2005}
}

@misc{Singhal:2005,
    title={Introducing the Knowledge Graph: things, not strings},
    author={Amit Singhal},
    publisher={Official Google Blog},
    year={2005}
}

@inproceedings{banarescu-et-al:2013AMR,
    title = "{A}bstract {M}eaning {R}epresentation for Sembanking",
    author="Banarescu, Laura and Bonial, Claire and Cai, Shu and Georgescu, Madalina and Griffitt, Kira and Hermjakob, Ulf and Knight, Kevin and Koehn, Philipp and Palmer, Martha and Schneider, Nathan",
    booktitle = "Proceedings of the 7th Linguistic Annotation Workshop and Interoperability with Discourse",
    year = "2013",
    pages = "178--186"
}

@article{Kulis:2013metric,
    year = {2013},
    volume = {5},
    journal = {Foundations and Trends\textregistered{} in Machine Learning},
    title = {Metric Learning: A Survey},
    number = {4},
    pages = {287-364},
    author = {Brian Kulis}
}

@inproceedings{pennington2014glove,
    author = {Jeffrey Pennington and Richard Socher and Christopher D. Manning},
    booktitle = {Proceedings of Empirical Methods in Natural Language Processing (EMNLP)},
    title = {GloVe: Global Vectors for Word Representation},
    year = {2014},
    pages = {1532--1543}
}

@inproceedings{Mikolov-et-al:2013distributed,
    author = {Mikolov, Tomas and Sutskever, Ilya and Chen, Kai and Corrado, Greg and Dean, Jeffrey},
    title = {Distributed Representations of Words and Phrases and Their Compositionality},
    year = {2013},
    booktitle = {Proceedings of the 26th International Conference on Neural Information Processing Systems - Volume 2},
    pages = {3111-3119}
}

@article{brown-etal:1993mathematics,
    title = "The Mathematics of Statistical Machine Translation: Parameter Estimation",
    author="Brown, Peter F. and Della Pietra, Stephen A. and Della Pietra, Vincent J. and Mercer, Robert L.",
    journal = "Computational Linguistics",
    volume = "19",
    number = "2",
    year = "1993",
    pages = "263--311"
}

@inproceedings{berg-kirkpatrick-etal:2012empirical,
    title = "An Empirical Investigation of Statistical Significance in {NLP}",
    author="Berg-Kirkpatrick, Taylor and Burkett, David and Klein, Dan",
    booktitle = "Proceedings of the 2012 Joint Conference on Empirical Methods in Natural Language Processing and Computational Natural Language Learning",
    year = "2012",
    pages = "995--1005"
}

@book{Sutton-and-Barto:2018RL,
    title={Reinforcement Learning: An Introduction (2nd ed.)},
    author={Richard S. Sutton and Andrew G. Barto},
    publisher={The MIT Press},
    year={2018}
}

@book{Gelman-etal:2020bayesian,
    author="Andrew Gelman and John B. Carlin and Hal S. Stern and David B. Dunson and Aki Vehtari and Donald B. Rubin",
    title="Bayesian Data Analysis (2nd ed.)",
    publisher="Chapman and Hall/CRC",
    year="2020"
}

@book{McElreath:2020rethinking,
    author="Richard McElreath",
    title="Statistical Rethinking: A Bayesian Course with Examples in R and STAN  (2nd ed.)",
    publisher="Chapman and Hall/CRC",
    year="2020"
}

@book{Downey:2021think,
    author="Allen B. Downey",
    title="Think Bayes: Bayesian Statistics in Python (2nd ed.)",
    publisher="O'Reilly Media",
    year="2021"
}

@article{Tay-etal:2020efficient,
    author  = {Yi Tay and Mostafa Dehghani and Dara Bahri and Donald Metzler},
    title = {Efficient Transformers: {A} Survey},
    journal = {CoRR},
    volume = {abs/2009.06732},
    year = {2020}
}

@article{xiao-and-zhu:2023introduction,
    title={Introduction to Transformers: an NLP Perspective},
    author={Xiao, Tong and Zhu, Jingbo},
    journal={arXiv preprint arXiv:2311.17633},
    year={2023}
}

@article{zhang-and-yang:2021multitask,
    author={Zhang, Yu and Yang, Qiang},
    journal={IEEE Transactions on Knowledge and Data Engineering},
    title={A Survey on Multi-Task Learning},
    year={2021},
    volume={},
    number={},
    pages={1-1}
}

@article{Wang-etal:2019ASO,
    title={A Survey of Zero-Shot Learning},
    author={Wei Wang and Vincent Wenchen Zheng and Han Yu and Chunyan Miao},
    journal={ACM Transactions on Intelligent Systems and Technology (TIST)},
    year={2019},
    volume={10},
    pages={1-37}
}

@article{wang-etal:2020fewshot,
    author = {Wang, Yaqing and Yao, Quanming and Kwok, James T. and Ni, Lionel M.},
    title = {Generalizing from a Few Examples: A Survey on Few-Shot Learning},
    year = {2020},
    publisher = {Association for Computing Machinery},
    volume = {53},
    number = {3},
    journal = {ACM Computing Surveys},
    articleno = {63},
    pages= {1-34}
}

@article{bahdanau2014neural,
    title={Neural machine translation by jointly learning to align and translate},
    author={Bahdanau, Dzmitry and Cho, Kyunghyun and Bengio, Yoshua},
    journal={arXiv preprint arXiv:1409.0473},
    year={2014}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CHAPTER 3

﻿@article{McCulloch-and-Pitts:1943,
    author={McCulloch, Warren S. and Pitts, Walter},
    title={A logical calculus of the ideas immanent in nervous activity},
    journal={The bulletin of mathematical biophysics},
    year={1943},
    volume={5},
    number={4},
    pages={115-133}
}

@article{hinton-etal:2006fast,
  title={A fast learning algorithm for deep belief nets},
  author={Hinton, Geoffrey E and Osindero, Simon and Teh, Yee-Whye},
  journal={Neural computation},
  volume={18},
  number={7},
  pages={1527--1554},
  year={2006}
}

@article{hinton2007learning,
  title={Learning multiple layers of representation},
  author={Hinton, Geoffrey E},
  journal={Trends in cognitive sciences},
  volume={11},
  number={10},
  pages={428--434},
  year={2007}
}

@article{lecun-etal:2015deep,
  title={Deep learning},
  author={LeCun, Yann and Bengio, Yoshua and Hinton, Geoffrey},
  journal={nature},
  volume={521},
  number={7553},
  pages={436--444},
  year={2015},
  publisher={Nature Publishing Group}
}

@book{Goodfellow-etal:2016,
    title={Deep Learning},
    author={Ian Goodfellow and Yoshua Bengio and Aaron Courville},
    publisher={MIT Press},
    year={2016}
}

@book{rosenblatt:1957perceptron,
    title={The perceptron, a perceiving and recognizing automaton Project Para},
    author={Rosenblatt, Frank},
    year={1957},
    publisher={Cornell Aeronautical Laboratory}
}

@book{minsky-and-Papert:1969perceptrons,
    title={Perceptrons},
    author={Minsky, Marvin and Papert, Seymour},
    year={1969},
    publisher={MIT press}
}

@article{cybenko:1989approximation,
    title={Approximation by superpositions of a sigmoidal function},
    author={Cybenko, George},
    journal={Mathematics of control, signals and systems},
    volume={2},
    number={4},
    pages={303--314},
    year={1989},
    publisher={Springer}
}

@article{bengio-etal:2003neural,
    title={A Neural Probabilistic Language Model},
    author={Bengio, Yoshua and Ducharme, R{\'e}jean and Vincent, Pascal and Jauvin, Christian},
    journal={Journal of Machine Learning Research},
    volume={3},
    number={Feb},
    pages={1137--1155},
    year={2003}
}

@article{hopfield:1984neurons,
    title={Neurons with graded response have collective computational properties like those of two-state neurons},
    author={Hopfield, John J},
    journal={Proceedings of the national academy of sciences},
    volume={81},
    number={10},
    pages={3088--3092},
    year={1984},
    publisher={National Acad Sciences}
}

@article{rumelhart-etal:1986learning,
    title={Learning representations by back-propagating errors},
    author={Rumelhart, David E and Hinton, Geoffrey E and Williams, Ronald J},
    journal={nature},
    volume={323},
    number={6088},
    pages={533--536},
    year={1986},
    publisher={Nature Publishing Group}
}

@article{williams-zipser:1989learning,
    title={A learning algorithm for continually running fully recurrent neural networks},
    author={Williams, Ronald J and Zipser, David},
    journal={Neural computation},
    volume={1},
    number={2},
    pages={270--280},
    year={1989},
    publisher={MIT Press}
}

@article{elman:1990finding,
    title={Finding structure in time},
    author={Elman, Jeffrey L},
    journal={Cognitive science},
    volume={14},
    number={2},
    pages={179--211},
    year={1990},
    publisher={Wiley Online Library}
}

@article{hubel-wiesel:1959receptive,
    title={Receptive fields of single neurones in the cat's striate cortex},
    author={Hubel, David H and Wiesel, Torsten N},
    journal={The Journal of physiology},
    volume={148},
    number={3},
    pages={574},
    year={1959},
    publisher={Wiley-Blackwell}
}

@article{waibel-etal:1989phoneme,
    title={Phoneme recognition using time-delay neural networks},
    author={Waibel, Alex and Hanazawa, Toshiyuki and Hinton, Geoffrey and Shikano, Kiyohiro and Lang, Kevin J},
    journal={IEEE transactions on acoustics, speech, and signal processing},
    volume={37},
    number={3},
    pages={328--339},
    year={1989},
    publisher={IEEE}
}

@article{lecun-etal:1989backpropagation,
    title={Backpropagation applied to handwritten zip code recognition},
    author={LeCun, Yann and Boser, Bernhard and Denker, John S and Henderson, Donnie and Howard, Richard E and Hubbard, Wayne and Jackel, Lawrence D},
    journal={Neural computation},
    volume={1},
    number={4},
    pages={541--551},
    year={1989},
    publisher={MIT Press}
}

@article{hochreiter-schmidhuber:1997long,
    title={Long short-term memory},
    author={Hochreiter, Sepp and Schmidhuber, J{\"u}rgen},
    journal={Neural computation},
    volume={9},
    number={8},
    pages={1735--1780},
    year={1997},
    publisher={MIT Press}
}

@inproceedings{Ioffe-and-Szegedy:2015batch,
    title={Batch normalization: Accelerating deep network training by reducing internal covariate shift},
    author={Ioffe, Sergey and Szegedy, Christian},
    booktitle={International conference on machine learning},
    pages={448--456},
    year={2015},
    organization={PMLR}
}

@article{Chiang-and-Cholak:2022overcoming,
    title={Overcoming a Theoretical Limitation of Self-Attention},
    author={Chiang, David and Cholak, Peter},
    journal={arXiv preprint arXiv:2202.12172},
    year={2022}
}

@article{Ba-etal:2016layer,
    title={Layer normalization},
    author={Ba, Jimmy Lei and Kiros, Jamie Ryan and Hinton, Geoffrey E},
    journal={arXiv preprint arXiv:1607.06450},
    year={2016}
}

@inproceedings{Wu-He:2018group,
    title={Group normalization},
    author={Wu, Yuxin and He, Kaiming},
    booktitle={Proceedings of the European conference on computer vision (ECCV)},
    pages={3--19},
    year={2018}
}

@article{Ulyanov-etal:2016instance,
    title={Instance normalization: The missing ingredient for fast stylization},
    author={Ulyanov, Dmitry and Vedaldi, Andrea and Lempitsky, Victor},
    journal={arXiv preprint arXiv:1607.08022},
    year={2016}
}

@inproceedings{he-etal:2016deep,
    title={Deep residual learning for image recognition},
    author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
    booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
    pages={770--778},
    year={2016}
}

@article{polyak:1964some,
    title={Some methods of speeding up the convergence of iteration methods},
    author={Polyak, Boris T},
    journal={Ussr computational mathematics and mathematical physics},
    volume={4},
    number={5},
    pages={1--17},
    year={1964},
    publisher={Elsevier}
}

@inproceedings{nesterov:1983method,
    title={A method for solving the convex programming problem with convergence rate O (1/k\^{} 2)},
    author={Nesterov, Yurii E},
    booktitle={Dokl. akad. nauk Sssr},
    volume={269},
    pages={543--547},
    year={1983}
}

@inproceedings{sutskever-etal:2013importance,
    title={On the importance of initialization and momentum in deep learning},
    author={Sutskever, Ilya and Martens, James and Dahl, George and Hinton, Geoffrey},
    booktitle={International conference on machine learning},
    pages={1139--1147},
    year={2013},
    organization={PMLR}
}

@article{zeiler:2012adadelta,
    title={Adadelta: an adaptive learning rate method},
    author={Zeiler, Matthew D},
    journal={arXiv preprint arXiv:1212.5701},
    year={2012}
}

@article{kingma-ba:2014adam,
    title={Adam: A method for stochastic optimization},
    author={Kingma, Diederik P and Ba, Jimmy},
    journal={arXiv preprint arXiv:1412.6980},
    year={2014}
}

@book{Kochenderfer-Wheeler:2019,
    title={Algorithms for Optimization},
    author={Mykel J. Kochenderfer and Tim A. Wheeler},
    publisher={The MIT Press},
    year={2019}
}

@inproceedings{glorot-bengio:2010understanding,
    title={Understanding the difficulty of training deep feedforward neural networks},
    author={Glorot, Xavier and Bengio, Yoshua},
    booktitle={Proceedings of the thirteenth international conference on artificial intelligence and statistics},
    pages={249--256},
    year={2010},
    organization={JMLR Workshop and Conference Proceedings}
}

@incollection{lecun-etal:2012efficient,
    title={Efficient backprop},
    author={LeCun, Yann A and Bottou, L{\'e}on and Orr, Genevieve B and M{\"u}ller, Klaus-Robert},
    booktitle={Neural networks: Tricks of the trade},
    pages={9--48},
    year={2012},
    publisher={Springer}
}

@inproceedings{he-etal:2015delving,
    title={Delving deep into rectifiers: Surpassing human-level performance on imagenet classification},
    author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
    booktitle={Proceedings of the IEEE international conference on computer vision},
    pages={1026--1034},
    year={2015}
}

@inproceedings{xu-etal:2020lipschitz,
    title = "Lipschitz Constrained Parameter Initialization for Deep Transformers",
    author = "Xu, Hongfei  and Liu, Qiuhui  and van Genabith, Josef  and Xiong, Deyi  and Zhang, Jingyi",
    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
    month = jul,
    year = "2020",
    pages = "397--402"
}

@inproceedings{huang-etal:2020improving,
    title={Improving transformer optimization through better initialization},
    author={Huang, Xiao Shi and Perez, Felipe and Ba, Jimmy and Volkovs, Maksims},
    booktitle={Proceedings of International Conference on Machine Learning},
    pages={4475--4483},
    year={2020},
    organization={PMLR}
}

@inproceedings{liu-etal:2020understanding,
    title = "Understanding the Difficulty of Training Transformers",
    author = "Liu, Liyuan  and Liu, Xiaodong  and Gao, Jianfeng  and Chen, Weizhu  and Han, Jiawei",
    booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
    month = nov,
    year = "2020",
    pages = "5747--5763"
}

@inproceedings{Vaswani-etal:2017Transformer,
    author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, \L{}ukasz and Polosukhin, Illia},
    booktitle = {Proceedings of Advances in Neural Information Processing Systems},
    title = {Attention is All you Need},
    volume = {30},
    year = {2017}
}

@inproceedings{ott-etal:2018scaling,
    title = "Scaling Neural Machine Translation",
    author = "Ott, Myle  and Edunov, Sergey  and Grangier, David  and Auli, Michael",
    booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers",
    month = oct,
    year = "2018",
    pages = "1--9"
}

@inproceedings{Smith-et-al:2018dontdecay,
    author="Samuel L. Smith and Pieter-Jan Kindermans and Chris Ying and Quoc V. Le",
    title     = {Don't Decay the Learning Rate, Increase the Batch Size},
    booktitle = {Proceedings of the 6th International Conference on Learning Representations {ICLR}},
    year      = {2018}
}

@article{srivastava-atal:2014dropout,
    title={Dropout: a simple way to prevent neural networks from overfitting},
    author={Srivastava, Nitish and Hinton, Geoffrey and Krizhevsky, Alex and Sutskever, Ilya and Salakhutdinov, Ruslan},
    journal={The journal of machine learning research},
    volume={15},
    number={1},
    pages={1929--1958},
    year={2014},
    publisher={JMLR. org}
}

@inproceedings{bishop:1995regularization,
    author = {Bishop, Christopher},
    title = {Regularization and Complexity Control in Feed-forward Networks},
    booktitle = {Proceedings International Conference on Artificial Neural Networks ICANN'95},
    year = {1995},
    pages = {141-148}
}

@article{Yao-etal:2007OnES,
    title={On Early Stopping in Gradient Descent Learning},
    author={Yuan Yao and Lorenzo Rosasco and Andrea Caponnetto},
    journal={Constructive Approximation},
    year={2007},
    volume={26},
    pages={289-315}
}

@InCollection{Jos:2017Boltzmann,
	  author       =	{Uffink, Jos},
	  title        =	{{Boltzmann's Work in Statistical Physics}},
	  booktitle    =	{The {Stanford} Encyclopedia of Philosophy},
	  editor       =	{Edward N. Zalta},
	  howpublished =	{\url{https://plato.stanford.edu/archives/spr2017/entries/statphys-Boltzmann/}},
	  year         =	{2017},
	  edition      =	{{S}pring 2017},
	  publisher    =	{Metaphysics Research Lab, Stanford University}
}

@inproceedings{szegedy-etal:2016rethinking,
    title={Rethinking the inception architecture for computer vision},
    author={Szegedy, Christian and Vanhoucke, Vincent and Ioffe, Sergey and Shlens, Jon and Wojna, Zbigniew},
    booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
    pages={2818--2826},
    year={2016}
}

@book{bickel-Doksum:2015mathematical,
    title={Mathematical statistics: basic ideas and selected topics, volumes I-II package},
    author={Bickel, Peter J and Doksum, Kjell A},
    year={2015},
    publisher={Chapman and Hall/CRC}
}

@inproceedings{goodman:1996-parsing,
    title = "Parsing Algorithms and Metrics",
    author = "Goodman, Joshua",
    booktitle = "34th Annual Meeting of the Association for Computational Linguistics",
    year = "1996",
    pages = "177--183"
}

@inproceedings{kumar-byrne:2004minimum,
    title = "Minimum {B}ayes-Risk Decoding for Statistical Machine Translation",
    author = "Kumar, Shankar  and Byrne, William",
    booktitle = "Proceedings of the Human Language Technology Conference of the North {A}merican Chapter of the Association for Computational Linguistics: {HLT}-{NAACL} 2004",
    year = "2004",
    pages = "169--176"
}

@misc{Olive:2022robust,
    title={Robust Statistics},
    author={David Olive},
    url={http://parker.ad.siu.edu/Olive/ol-bookp.htm},
    year={2022}
}

@inproceedings{Gulcehre-etal:2016noisy,
    title = {Noisy Activation Functions},
    author = {Gulcehre, Caglar and Moczulski, Marcin and Denil, Misha and Bengio, Yoshua},
    booktitle = {Proceedings of The 33rd International Conference on Machine Learning},
    pages = {3059--3068},
    year = {2016},
    editor = {Balcan, Maria Florina and Weinberger, Kilian Q.},
    volume = {48},
    series = {Proceedings of Machine Learning Research},
    publisher = {PMLR}
}

@article{zhang-etal:2020adversarial,
    title={Adversarial attacks on deep-learning models in natural language processing: A survey},
    author={Zhang, Wei Emma and Sheng, Quan Z and Alhazmi, Ahoud and Li, Chenliang},
    journal={ACM Transactions on Intelligent Systems and Technology (TIST)},
    volume={11},
    number={3},
    pages={1--41},
    year={2020},
    publisher={ACM New York, NY, USA}
}

@inproceedings{jia-liang:2017adversarial,
    title = "Adversarial Examples for Evaluating Reading Comprehension Systems",
    author = "Jia, Robin  and Liang, Percy",
    booktitle = "Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing",
    year = "2017",
    pages = "2021--2031"
}

@inproceedings{belinkov-bisk:2018synthetic,
    title={Synthetic and Natural Noise Both Break Neural Machine Translation},
    author={Belinkov, Yonatan and Bisk, Yonatan},
    booktitle={International Conference on Learning Representations},
    year={2018}
}

@inproceedings{ebrahimi-etal:2018hotflip,
    title = "{H}ot{F}lip: White-Box Adversarial Examples for Text Classification",
    author="Ebrahimi, Javid and Rao, Anyi and Lowd, Daniel and Dou, Dejing",
    booktitle = "Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)",
    year = "2018",
    address = "Melbourne, Australia",
    pages = "31--36"
}

@inproceedings{alzantot-etal:2018generating,
    title={Generating Natural Language Adversarial Examples},
    author={Alzantot, Moustafa and Sharma, Yash and Elgohary, Ahmed and Ho, Bo-Jhang and Srivastava, Mani and Chang, Kai-Wei},
    booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
    pages={2890--2896},
    year={2018}
}

@article{pearson:1901PCA,
    title={On lines and planes of closest fit to systems of points in space},
    author={Pearson, Karl},
    journal={The London, Edinburgh, and Dublin philosophical magazine and journal of science},
    volume={2},
    number={11},
    pages={559--572},
    year={1901},
    publisher={Taylor {\&} Francis}
}

@article{wold-etal:1987principal,
    title={Principal component analysis},
    author={Wold, Svante and Esbensen, Kim and Geladi, Paul},
    journal={Chemometrics and intelligent laboratory systems},
    volume={2},
    number={1-3},
    pages={37--52},
    year={1987},
    publisher={Elsevier}
}

@article{hinton-Salakhutdinov:2006reducing,
    title={Reducing the dimensionality of data with neural networks},
    author={Hinton, Geoffrey E and Salakhutdinov, Ruslan R},
    journal={science},
    volume={313},
    number={5786},
    pages={504--507},
    year={2006},
    publisher={American Association for the Advancement of Science}
}

@article{ackley-etal:1985learning,
    title={A learning algorithm for Boltzmann machines},
    author={Ackley, David H and Hinton, Geoffrey E and Sejnowski, Terrence J},
    journal={Cognitive science},
    volume={9},
    number={1},
    pages={147--169},
    year={1985},
    publisher={Elsevier}
}

@article{dayan-etal:1995helmholtz,
    title={The helmholtz machine},
    author={Dayan, Peter and Hinton, Geoffrey E and Neal, Radford M and Zemel, Richard S},
    journal={Neural computation},
    volume={7},
    number={5},
    pages={889--904},
    year={1995}
}

@article{hopfield1982neural,
    title={Neural networks and physical systems with emergent collective computational abilities},
    author={Hopfield, John J},
    journal={Proceedings of the national academy of sciences},
    volume={79},
    number={8},
    pages={2554--2558},
    year={1982},
    publisher={National Acad Sciences}
}

@article{olshausen-field:1997sparse,
    title={Sparse coding with an overcomplete basis set: A strategy employed by V1?},
    author={Olshausen, Bruno A and Field, David J},
    journal={Vision research},
    volume={37},
    number={23},
    pages={3311--3325},
    year={1997},
    publisher={Elsevier}
}

@article{nair-hinton:20093d,
    title={3D object recognition with deep belief nets},
    author={Nair, Vinod and Hinton, Geoffrey E},
    journal={Advances in neural information processing systems},
    volume={22},
    year={2009}
}

@inproceedings{rifai-etal:11contractiveauto-encoders,
    author = {Salah Rifai and Pascal Vincent and Xavier Muller and Xavier Glorot and Yoshua Bengio},
    title = {Contractive auto-encoders: Explicit invariance during feature extraction},
    booktitle = {Proceedings of International Conference on Machine Learning},
    year = {2011}
}

@article{vincent-etal:2010stacked,
    title={Stacked denoising autoencoders: Learning useful representations in a deep network with a local denoising criterion.},
    author={Vincent, Pascal and Larochelle, Hugo and Lajoie, Isabelle and Bengio, Yoshua and Manzagol, Pierre-Antoine and Bottou, L{\'e}on},
    journal={Journal of machine learning research},
    volume={11},
    number={12},
    year={2010}
}

@inproceedings{Kingma-Welling:2014autoencoding,
    author = {Kingma, Diederik P. and Welling, Max},
    booktitle = {Proceedings of 2nd International Conference on Learning Representations, {ICLR} 2014},
    title = {Auto-Encoding Variational Bayes},
    year = 2014
}

@article{kingma-welling:2019,
    title={An Introduction to Variational Autoencoders},
    author={Diederik P. Kingma and Max Welling},
    journal={Foundations and Trends\textregistered{} in Machine Learning},
    publisher={Now Publishers Inc },
    year={2019}
}

@article{doersch:2016tutorial,
    title={Tutorial on Variational Autoencoders},
    author={Doersch, Carl},
    journal={stat},
    volume={1050},
    pages={13},
    year={2016}
}

@book{barber:2012,
    title={Bayesian Reasoning and Machine Learning},
    publisher={Cambridge University Press},
    author={Barber, David},
    year={2012}
}

@misc{knight:2009bayesian,
    title={Bayesian inference with tears},
    author={Knight, Kevin},
    booktitle={Tutorial workbook},
    year={2009}
}

@article{zhang-etal:2021dive,
    title={Dive into Deep Learning},
    author={Zhang, Aston and Lipton, Zachary C. and Li, Mu and Smola, Alexander J.},
    journal={arXiv preprint arXiv:2106.11342},
    year={2021}
}

@book{Chollet:2021dl,
    title={Deep Learning with Python (2nd ed.)},
    author={Fran\c{c}ois Chollet},
    publisher={Manning Publications},
    year={2021}
}

@book{Geron:2019,
    title={Hands-On Machine Learning with Scikit-Learn, Keras, and TensorFlow: Concepts, Tools, and Techniques to Build Intelligent Systems (2nd ed.)},
    author={Aur\'elien G\'eron},
    publisher={O'Reilly Media},
    year={2019}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CHAPTER 4

@book{jackendoff:1992semantic,
    title={Semantic structures},
    author={Jackendoff, Ray S},
    volume={18},
    year={1992},
    publisher={MIT press}
}

@book{aronoff-fudeman:2011morphology,
    title={What is morphology?},
    author={Aronoff, Mark and Fudeman, Kirsten},
    volume={8},
    year={2011},
    publisher={John Wiley \& Sons}
}

@book{Lawson:2003,
    title={Finite Automata (1st ed.)},
    author={Mark V. Lawson},
    publisher={Chapman and Hall/CRC},
    year={2003}
}

@book{Friedl:2006,
    title={Mastering Regular Expressions (3rd ed.)},
    author={Jeffrey Friedl},
    publisher={O'Reilly Media},
    year={2006}
}

@inproceedings{koehn-etal:2007-moses,
    title = "{M}oses: Open Source Toolkit for Statistical Machine Translation",
    author="Koehn, Philipp and Hoang, Hieu and Birch, Alexandra and Callison-Burch, Chris and Federico, Marcello and Bertoldi, Nicola and Cowan, Brooke and Shen, Wade and Moran, Christine and Zens, Richard and Dyer, Chris and Bojar, Ond{\v{r}}ej and Constantin, Alexandra and Herbst, Evan",
    booktitle = "Proceedings of the 45th Annual Meeting of the Association for Computational Linguistics Companion Volume Proceedings of the Demo and Poster Sessions",
    year = "2007",
    pages = "177--180",
}

@article{porter:1980algorithm,
    title={An algorithm for suffix stripping},
    author={Porter, Martin F},
    journal={Program},
    year={1980},
    publisher={MCB UP Ltd}
}

@inproceedings{zhao-etal:2006effective,
    title = "Effective Tag Set Selection in {C}hinese Word Segmentation via Conditional Random Field Modeling",
    author="Zhao, Hai and Huang, Chang-Ning and Li, Mu and Lu, Bao-Liang",
    booktitle = "Proceedings of the 20th Pacific Asia Conference on Language, Information and Computation",
    year = "2006",
    pages = "87--94"
}

@inproceedings{McCallum-etal:2000MEMM,
    author = {McCallum, Andrew and Freitag, Dayne and Pereira, Fernando C. N.},
    title = {Maximum Entropy Markov Models for Information Extraction and Segmentation},
    year = {2000},
    booktitle = {Proceedings of the Seventeenth International Conference on Machine Learning},
    pages = {591-598}
}

@article{Kupiec:1992RobustPT,
    title={Robust part-of-speech tagging using a hidden Markov model},
    author={Julian Kupiec},
    journal={Computer Speech \& Language},
    year={1992},
    volume={6},
    pages={225-242}
}

@inproceedings{sennrich:2016neural,
    title={Neural Machine Translation of Rare Words with Subword Units},
    author={Sennrich, Rico and Haddow, Barry and Birch, Alexandra},
    booktitle={Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={1715--1725},
    year={2016}
}

@article{gage:1994new,
    title={A new algorithm for data compression},
    author={Gage, Philip},
    journal={C Users Journal},
    volume={12},
    number={2},
    pages={23--38},
    year={1994}
}

@inproceedings{Schuster-Nakajima:2012wordpiece,
    title	= {Japanese and Korean Voice Search},
    author	= {Mike Schuster and Kaisuke Nakajima},
    year	= {2012},
    booktitle	= {Proceedings of International Conference on Acoustics, Speech and Signal Processing},
    pages	= {5149--5152}
}

@inproceedings{kudo-richardson:2018sentencepiece,
    title={SentencePiece: A simple and language independent subword tokenizer and detokenizer for Neural Text Processing},
    author={Kudo, Taku and Richardson, John},
    booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing: System Demonstrations},
    pages={66--71},
    year={2018}
}

@inproceedings{kudo:2018subword,
    title={Subword Regularization: Improving Neural Network Translation Models with Multiple Subword Candidates},
    author={Kudo, Taku},
    booktitle={Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={66--75},
    year={2018}
}

@article{dempster-etal:1977maximum,
    title={Maximum likelihood from incomplete data via the EM algorithm},
    author={Dempster, Arthur P and Laird, Nan M and Rubin, Donald B},
    journal={Journal of the Royal Statistical Society: Series B (Methodological)},
    volume={39},
    number={1},
    pages={1--22},
    year={1977},
    publisher={Wiley Online Library}
}

@misc{knight:2018poetry,
    title={The Sparrow Tweets},
    author={Linda Knight},
    booktitle={A Picture of Poetry For Children},
    year={2018}
}

@InCollection{szabo:2020compositionality,
	  author = {Szab\'o, Zolt\'an Gendler},
	  title        =	{{Compositionality}},
	  booktitle    =	{The {Stanford} Encyclopedia of Philosophy},
	  editor       =	{Edward N. Zalta},
	  howpublished =	{\url{https://plato.stanford.edu/archives/fall2020/entries/compositionality/}},
	  year         =	{2020},
	  edition      =	{{F}all 2020},
	  publisher    =	{Metaphysics Research Lab, Stanford University}
}

@incollection{Montague:1974,
    title={Universal Grammar},
    author={Richard Montague},
    booktitle={Formal Philosophy: Selected Papers of Richard Montague},
    editor={R. Thomason},
    publisher={Yale University Press},
    year={1974}
}

@article{lund-burgess:1996producing,
    title={Producing high-dimensional semantic spaces from lexical co-occurrence},
    author={Lund, Kevin and Burgess, Curt},
    journal={Behavior research methods, instruments, \& computers},
    volume={28},
    number={2},
    pages={203--208},
    year={1996},
    publisher={Springer}
}

@article{church-hanks:1990word,
    title = "Word Association Norms, Mutual Information, and Lexicography",
    author = "Church, Kenneth Ward  and Hanks, Patrick",
    journal = "Computational Linguistics",
    volume = "16",
    number = "1",
    year = "1990",
    url = "https://aclanthology.org/J90-1003",
    pages = "22--29",
}

@book{buttcher-etal:2016information,
    title={Information retrieval: Implementing and evaluating search engines},
    author={Buttcher, Stefan and Clarke, Charles LA and Cormack, Gordon V},
    year={2016},
    publisher={MIT Press}
}

@book{Wright-Ma:2022,
    author = {John Wright and Yi Ma},
    title = {High-Dimensional Data Analysis with Low-Dimensional Models: Principles, Computation, and Applications},
    publisher = {Cambridge University Press},
    year = {2022}
}

@article{deerwester-etal:1990indexing,
    title={Indexing by latent semantic analysis},
    author={Deerwester, Scott and Dumais, Susan T and Furnas, George W and Landauer, Thomas K and Harshman, Richard},
    journal={Journal of the American society for information science},
    volume={41},
    number={6},
    pages={391--407},
    year={1990},
    publisher={Wiley Online Library}
}

@article{landauer-etal:1998introduction,
    title={An introduction to latent semantic analysis},
    author={Landauer, Thomas K and Foltz, Peter W and Laham, Darrell},
    journal={Discourse processes},
    volume={25},
    number={2-3},
    pages={259--284},
    year={1998},
    publisher={Taylor \& Francis}
}

@article{stewart:1993early,
    title={On the early history of the singular value decomposition},
    author={Stewart, Gilbert W},
    journal={SIAM review},
    volume={35},
    number={4},
    pages={551--566},
    year={1993},
    publisher={SIAM}
}

@incollection{Cline-Dhillon:2014,
    title={Computation of the Singular Value Decomposition},
    author={Alan Kaylor Cline and Inderjit S. Dhillon},
    booktitle={Handbook of Linear Algebra (2dn ed.)},
    editor={Leslie Hogben},
    publisher={CRC Press},
    year={2014}
}

@book{jolliffe:2002principal,
    title={Principal component analysis for special types of data},
    author={Jolliffe, Ian T},
    year={2002},
    publisher={Springer}
}

@article{blei-etal:2012probabilistic,
    title={Probabilistic topic models},
    author={Blei, David M},
    journal={Communications of the ACM},
    volume={55},
    number={4},
    pages={77--84},
    year={2012},
    publisher={ACM New York, NY, USA}
}

@article{blei-etal:2003latent,
    title={Latent dirichlet allocation},
    author={Blei, David M and Ng, Andrew Y and Jordan, Michael I},
    journal={Journal of machine Learning research},
    volume={3},
    number={Jan},
    pages={993--1022},
    year={2003}
}

@article{solorio-etal:2020review,
    title={A review of unsupervised feature selection methods},
    author={Solorio-Fern{\'a}ndez, Sa{\'u}l and Carrasco-Ochoa, J Ariel and Mart{\'\i}nez-Trinidad, Jos{\'e} Fco},
    journal={Artificial Intelligence Review},
    volume={53},
    number={2},
    pages={907--948},
    year={2020},
    publisher={Springer}
}

@article{guyon-elisseeff:2003introduction,
    title={An introduction to variable and feature selection},
    author={Guyon, Isabelle and Elisseeff, Andr{\'e}},
    journal={Journal of machine learning research},
    volume={3},
    number={Mar},
    pages={1157--1182},
    year={2003}
}

@book{liu-motoda:2012feature,
    title={Feature selection for knowledge discovery and data mining},
    author={Liu, Huan and Motoda, Hiroshi},
    volume={454},
    year={2012},
    publisher={Springer Science \& Business Media}
}

@inproceedings{mikolov:2013efficient,
    title={Efficient Estimation of Word Representations in Vector Space},
    author={Mikolov, Tomas and Chen, Kai and Corrado, Greg and Dean, Jeffrey},
    booktitle={Proceedings of the International Conference on Learning Representations (ICLR 2013)},
    year={2013}
}

@article{mnih-kavukcuoglu:2013learning,
    title={Learning word embeddings efficiently with noise-contrastive estimation},
    author={Mnih, Andriy and Kavukcuoglu, Koray},
    journal={Advances in neural information processing systems},
    volume={26},
    year={2013}
}

@inproceedings{joulin-etal:2017bag,
    title={Bag of Tricks for Efficient Text Classification},
    author={Joulin, Armand and Grave, {\'E}douard and Bojanowski, Piotr and Mikolov, Tom{\'a}{\v{s}}},
    booktitle={Proceedings of the 15th Conference of the European Chapter of the Association for Computational Linguistics: Volume 2, Short Papers},
    pages={427--431},
    year={2017}
}

@inproceedings{telgarsky:2016benefits,
    title={Benefits of depth in neural networks},
    author={Telgarsky, Matus},
    booktitle={Conference on learning theory},
    pages={1517--1539},
    year={2016},
    organization={PMLR}
}

@inproceedings{devlin-etal:2019bert,
  title={BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding},
  author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
  booktitle={Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)},
  pages={4171--4186},
  year={2019}
}

@inproceedings{pascanu-etal:2013difficulty,
    title={On the difficulty of training recurrent neural networks},
    author={Pascanu, Razvan and Mikolov, Tomas and Bengio, Yoshua},
    booktitle={International conference on machine learning},
    pages={1310--1318},
    year={2013},
    organization={PMLR}
}

@inproceedings{bapna-etal:2018training,
    title={Training Deeper Neural Machine Translation Models with Transparent Attention},
    author={Bapna, Ankur and Chen, Mia Xu and Firat, Orhan and Cao, Yuan and Wu, Yonghui},
    booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
    pages={3028--3033},
    year={2018}
}

@inproceedings{wang-etal:2019learning,
    title={Learning Deep Transformer Models for Machine Translation},
    author={Wang, Qiang and Li, Bei and Xiao, Tong and Zhu, Jingbo and Li, Changliang and Wong, Derek F and Chao, Lidia S},
    booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
    pages={1810--1822},
    year={2019}
}

@inproceedings{zhang-etal:2019improving,
    title={Improving Deep Transformer with Depth-Scaled Initialization and Merged Attention},
    author={Zhang, Biao and Titov, Ivan and Sennrich, Rico},
    booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
    pages={898--909},
    year={2019}
}

@article{pham-etal:2019very,
    title={Very deep self-attention networks for end-to-end speech recognition},
    author={Pham, Ngoc-Quan and Nguyen, Thai-Son and Niehues, Jan and M{\"u}ller, Markus and St{\"u}ker, Sebastian and Waibel, Alexander},
    journal={arXiv preprint arXiv:1904.13377},
    year={2019}
}

@inproceedings{li-etal:2020shallow,
    title={Shallow-to-Deep Training for Neural Machine Translation},
    author={Li, Bei and Wang, Ziyang and Liu, Hui and Jiang, Yufan and Du, Quan and Xiao, Tong and Wang, Huizhen and Zhu, Jingbo},
    booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
    pages={995--1005},
    year={2020}
}

@inproceedings{levy-goldberg:2014linguistic,
    title={Linguistic Regularities in sparse and explicit word representations},
    author={Levy, Omer and Goldberg, Yoav},
    booktitle={Proceedings of the eighteenth conference on computational natural language learning},
    pages={171--180},
    year={2014}
}

@article{levy-goldberg:2014neural,
    title={Neural word embedding as implicit matrix factorization},
    author={Levy, Omer and Goldberg, Yoav},
    journal={Advances in neural information processing systems},
    volume={27},
    year={2014}
}

@inproceedings{linzen:2016issues,
    title = "Issues in evaluating semantic spaces using word analogies",
    author = "Linzen, Tal",
    booktitle = "Proceedings of the 1st Workshop on Evaluating Vector-Space Representations for {NLP}",
    year = "2016",
    pages = "13--18",
}

@inproceedings{schnabel-etal:2015evaluation,
    title={Evaluation methods for unsupervised word embeddings},
    author={Schnabel, Tobias and Labutov, Igor and Mimno, David and Joachims, Thorsten},
    booktitle={Proceedings of the 2015 conference on empirical methods in natural language processing},
    pages={298--307},
    year={2015}
}

@article{levy-etal:2015improving,
    title={Improving distributional similarity with lessons learned from word embeddings},
    author={Levy, Omer and Goldberg, Yoav and Dagan, Ido},
    journal={Transactions of the association for computational linguistics},
    volume={3},
    pages={211--225},
    year={2015},
    publisher={MIT Press}
}

@inproceedings{gladkova-etal:2016analogy,
    title={Analogy-based detection of morphological and semantic relations with word embeddings: what works and what doesn't.},
    author={Gladkova, Anna and Drozd, Aleksandr and Matsuoka, Satoshi},
    booktitle={Proceedings of the NAACL Student Research Workshop},
    pages={8--15},
    year={2016}
}

@inproceedings{schnabel:2015evaluation,
  title={Evaluation methods for unsupervised word embeddings},
  author={Schnabel, Tobias and Labutov, Igor and Mimno, David and Joachims, Thorsten},
  booktitle={Proceedings of the 2015 conference on empirical methods in natural language processing},
  pages={298--307},
  year={2015}
}

@article{bakarov:2018survey,
    title={A survey of word embeddings evaluation methods},
    author={Bakarov, Amir},
    journal={arXiv preprint arXiv:1801.09536},
    year={2018}
}

@inproceedings{rogers-etal:2018what,
    title={What's in your embedding, and how it predicts task performance},
    author={Rogers, Anna and Ananthakrishna, Shashwath Hosur and Rumshisky, Anna},
    booktitle={Proceedings of the 27th International Conference on Computational Linguistics},
    pages={2690--2703},
    year={2018}
}

@article{rubenstein-goodenough:1965contextual,
    title={Contextual correlates of synonymy},
    author={Rubenstein, Herbert and Goodenough, John B},
    journal={Communications of the ACM},
    volume={8},
    number={10},
    pages={627--633},
    year={1965},
    publisher={ACM New York, NY, USA}
}

@inproceedings{reisinger-mooney:2010multi,
    title={Multi-prototype vector-space models of word meaning},
    author={Reisinger, Joseph and Mooney, Raymond},
    booktitle={Human Language Technologies: The 2010 Annual Conference of the North American Chapter of the Association for Computational Linguistics},
    pages={109--117},
    year={2010}
}

@inproceedings{faruqui-etal:2016problems,
    title={Problems With Evaluation of Word Embeddings Using Word Similarity Tasks},
    author={Faruqui, Manaal and Tsvetkov, Yulia and Rastogi, Pushpendre and Dyer, Chris},
    booktitle={Proceedings of the 1st Workshop on Evaluating Vector-Space Representations for NLP},
    pages={30--35},
    year={2016}
}

@inproceedings{huang-etal:2012improving,
    title={Improving word representations via global context and multiple word prototypes},
    author={Huang, Eric H and Socher, Richard and Manning, Christopher D and Ng, Andrew Y},
    booktitle={Proceedings of the 50th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={873--882},
    year={2012}
}

@inproceedings{baroni-etal:2014dont,
    title = "Don{'}t count, predict! A systematic comparison of context-counting vs. context-predicting semantic vectors",
    author = "Baroni, Marco  and Dinu, Georgiana  and Kruszewski, Germ{\'a}n",
    booktitle = "Proceedings of the 52nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    year = "2014",
    pages = "238--247",
}

@inproceedings{mikolov-etal:2013linguistic,
    title={Linguistic regularities in continuous space word representations},
    author={Mikolov, Tomas and Yih, Wen-tau and Zweig, Geoffrey},
    booktitle={Proceedings of the 2013 conference of the north american chapter of the association for computational linguistics: Human language technologies},
    pages={746--751},
    year={2013}
}

@inproceedings{allen-hospedales:2019analogies,
    title={Analogies explained: Towards understanding word embeddings},
    author={Allen, Carl and Hospedales, Timothy},
    booktitle={International Conference on Machine Learning},
    pages={223--231},
    year={2019},
    organization={PMLR}
}

@article{schacter-buckner:1998priming,
    title={Priming and the brain},
    author={Schacter, Daniel L and Buckner, Randy L},
    journal={Neuron},
    volume={20},
    number={2},
    pages={185--195},
    year={1998},
    publisher={Elsevier}
}

@article{tulving-schacter:1990priming,
    title={Priming and human memory systems},
    author={Tulving, Endel and Schacter, Daniel L},
    journal={Science},
    volume={247},
    number={4940},
    pages={301--306},
    year={1990},
    publisher={American Association for the Advancement of Science}
}

@article{wiggs-martin:1998properties,
    title={Properties and mechanisms of perceptual priming},
    author={Wiggs, Cheri L and Martin, Alex},
    journal={Current opinion in neurobiology},
    volume={8},
    number={2},
    pages={227--233},
    year={1998},
    publisher={Elsevier}
}

@article{meyer-schvaneveldt:1971facilitation,
    title={Facilitation in recognizing pairs of words: evidence of a dependence between retrieval operations.},
    author={Meyer, David E and Schvaneveldt, Roger W},
    journal={Journal of experimental psychology},
    volume={90},
    number={2},
    pages={227},
    year={1971},
    publisher={American Psychological Association}
}

@inproceedings{lund:1995semantic,
    title={Semantic and associative priming in high-dimensional semantic space},
    author={Lund, Kevin},
    booktitle={Proc. of the 17th Annual conferences of the Cognitive Science Society, 1995},
    year={1995}
}

@book{mcnamara:2005semantic,
    title={Semantic priming: Perspectives from memory and word recognition},
    author={McNamara, Timothy P},
    year={2005},
    publisher={Psychology Press}
}

@article{hutchison-etal:2013semantic,
    title={The semantic priming project},
    author={Hutchison, Keith A and Balota, David A and Neely, James H and Cortese, Michael J and Cohen-Shikora, Emily R and Tse, Chi-Shing and Yap, Melvin J and Bengson, Jesse J and Niemeyer, Dale and Buchanan, Erin},
    journal={Behavior research methods},
    volume={45},
    number={4},
    pages={1099--1114},
    year={2013},
    publisher={Springer}
}

@article{lapesa-evert:2014large,
    title={A large scale evaluation of distributional semantic models: Parameters, interactions and model selection},
    author={Lapesa, Gabriella and Evert, Stefan},
    journal={Transactions of the Association for Computational Linguistics},
    volume={2},
    pages={531--546},
    year={2014},
    publisher={MIT Press}
}

@inproceedings{sogaard:2016evaluating,
    title={Evaluating word embeddings with fMRI and eye-tracking},
    author={S{\o}gaard, Anders},
    booktitle={Proceedings of the 1st workshop on evaluating vector-space representations for NLP},
    pages={116--121},
    year={2016}
}

@inproceedings{klerke-etal:2015looking,
    title={Looking hard: Eye tracking for detecting grammaticality of automatically compressed sentences},
    author={Klerke, Sigrid and Alonso, H{\'e}ctor Mart{\'\i}nez and S{\o}gaard, Anders},
    booktitle={Proceedings of the 20th Nordic Conference of Computational Linguistics (NODALIDA 2015)},
    pages={97--105},
   year={2015}
}

@inproceedings{auguste-etal:2017evaluation,
    title={Evaluation of word embeddings against cognitive processes: primed reaction times in lexical decision and naming tasks},
    author={Auguste, Jeremy and Rey, Arnaud and Favre, Benoit},
    booktitle={Proceedings of the 2nd workshop on evaluating vector space representations for NLP},
    pages={21--26},
    year={2017}
}

@inproceedings{devereux-etal:2010using,
    title={Using fMRI activation to conceptual stimuli to evaluate methods for extracting conceptual representations from corpora},
    author={Devereux, Barry and Kelly, Colin and Korhonen, Anna},
    booktitle={Proceedings of the NAACL HLT 2010 First Workshop on Computational Neurolinguistics},
    pages={70--78},
    year={2010}
}

@inproceedings{bhattasali-etal:2020alice,
    title = "The Alice Datasets: f{MRI} {\&} {EEG} Observations of Natural Language Comprehension",
    author = "Bhattasali, Shohini  and Brennan, Jonathan  and Luh, Wen-Ming  and Franzluebbers, Berta  and Hale, John",
    booktitle = "Proceedings of the 12th Language Resources and Evaluation Conference",
    year = "2020",
    pages = "120--125",
}

@inproceedings{tsvetkov:2015evaluation,
    title={Evaluation of word vector representations by subspace alignment},
    author={Tsvetkov, Yulia and Faruqui, Manaal and Ling, Wang and Lample, Guillaume and Dyer, Chris},
    booktitle={Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing},
    pages={2049--2054},
    year={2015}
}

@inproceedings{acs-kornai:2016evaluating,
    title={Evaluating embeddings on dictionary-based similarity},
    author={Acs, Judit and Kornai, Andr{\'a}s},
    booktitle={Proceedings of the 1st Workshop on Evaluating Vector-Space Representations for NLP},
    pages={78--82},
    year={2016}
}

@inproceedings{agirre-etal:2009study,
    title={A Study on Similarity and Relatedness Using Distributional and WordNet-based Approaches},
    author={Agirre, Eneko and Alfonseca, Enrique and Hall, Keith and Kravalov{\'a}, Jana and Pasca, Marius and Soroa, Aitor},
    booktitle={Proceedings of Human Language Technologies: The 2009 Annual Conference of the North American Chapter of the Association for Computational Linguistics},
    pages={19--27},
    year={2009}
}

@article{hinton-roweis:2002stochastic,
    title={Stochastic neighbor embedding},
    author={Hinton, Geoffrey E and Roweis, Sam},
    journal={Advances in neural information processing systems},
    volume={15},
    year={2002}
}

@article{van-hinton:2008visualizing,
    title={Visualizing data using t-SNE.},
    author={Van der Maaten, Laurens and Hinton, Geoffrey},
    journal={Journal of machine learning research},
    volume={9},
    number={11},
    year={2008}
}

@article{zhang-etal:2019word,
    title={Word embedding visualization via dictionary learning},
    author={Zhang, Juexiao and Chen, Yubei and Cheung, Brian and Olshausen, Bruno A},
    journal={arXiv preprint arXiv:1910.03833},
    year={2019}
}

@article{liu-etal:2017visual,
    title={Visual exploration of semantic relationships in neural word embeddings},
    author={Liu, Shusen and Bremer, Peer-Timo and Thiagarajan, Jayaraman J and Srikumar, Vivek and Wang, Bei and Livnat, Yarden and Pascucci, Valerio},
    journal={IEEE transactions on visualization and computer graphics},
    volume={24},
    number={1},
    pages={553--562},
    year={2017},
    publisher={IEEE}
}

@inproceedings{webster-kit:1992tokenization,
    title={Tokenization as the initial phase in NLP},
    author={Webster, Jonathan J and Kit, Chunyu},
    booktitle={Proceedings of COLING 1992 volume 4: The 14th international conference on computational linguistics},
    year={1992}
}

@article{mielke-etal:2021between,
    title={Between words and characters: A Brief History of Open-Vocabulary Modeling and Tokenization in NLP},
    author={Mielke, Sabrina J. and Alyafeai, Zaid and Salesky, Elizabeth and Raffel, Colin and Dey, Manan and Gall{\'e}, Matthias and Raja, Arun and Si, Chenglei and Lee, Wilson Y. and Sagot, Beno{\^i}t and Tan, Samson},
    journal={arXiv preprint arXiv:2112.10508},
    year={2021}
}

@inproceedings{ling-etal:2015finding,
    title={Finding Function in Form: Compositional Character Models for Open Vocabulary Word Representation},
    author={Ling, Wang and Dyer, Chris and Black, Alan W and Trancoso, Isabel and Fermandez, Ram{\'o}n and Amir, Silvio and Marujo, Luis and Lu{\'\i}s, Tiago},
    booktitle={Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing},
    pages={1520--1530},
    year={2015}
}

@article{lee-etal:2017fully,
    title={Fully Character-Level Neural Machine Translation without Explicit Segmentation},
    author={Lee, Jason and Cho, Kyunghyun and Hofmann, Thomas},
    journal={Transactions of the Association for Computational Linguistics},
    volume={5},
    pages={365--378},
    year={2017}
}

@inproceedings{schneider-etal:2019wav2vec,
    title={wav2vec: Unsupervised Pre-training for Speech Recognition},
    author={Steffen Schneider and Alexei Baevski and Ronan Collobert and Michael Auli},
    booktitle={INTERSPEECH},
    year={2019}
}

@inproceedings{dosovitskiy-etal:2020image,
    title={An image is worth 16x16 words: Transformers for image recognition at scale},
    author={Dosovitskiy, Alexey and Beyer, Lucas and Kolesnikov, Alexander and Weissenborn, Dirk and Zhai, Xiaohua and Unterthiner, Thomas and Dehghani, Mostafa and Minderer, Matthias and Heigold, Georg and Gelly, Sylvain and Uszkoreit, Jakob and Houlsby, Neil},
    booktitle={Proceedings of ICLR 2021},
    year={2021}
}

@article{van-etal:2017neural,
    title={Neural discrete representation learning},
    author={Oord, Aaron van den and Vinyals, Oriol and Kavukcuoglu, Koray},
    journal={Advances in neural information processing systems},
    volume={30},
    year={2017}
}

@inproceedings{baevski-etal:2019vq,
    title={vq-wav2vec: Self-supervised learning of discrete speech representations},
    author={Baevski, Alexei and Schneider, Steffen and Auli, Michael},
    booktitle={Proceedings of ICLR 2020},
    year={2020}
}

@article{hsu-etal:2021hubert,
    title={Hubert: Self-supervised speech representation learning by masked prediction of hidden units},
    author={Hsu, Wei-Ning and Bolte, Benjamin and Tsai, Yao-Hung Hubert and Lakhotia, Kushal and Salakhutdinov, Ruslan and Mohamed, Abdelrahman},
    journal={IEEE/ACM Transactions on Audio, Speech, and Language Processing},
    volume={29},
    pages={3451--3460},
    year={2021},
    publisher={IEEE}
}

@article{church:2011pendulum,
    title={A pendulum swung too far},
    author={Church, Kenneth},
    journal={Linguistic Issues in Language Technology},
    volume={6},
    year={2011}
}

@book{wittgenstein:1953philosophical,
    title={Philosophical investigations. Philosophische Untersuchungen.},
    author={Wittgenstein, Ludwig},
    year={1953},
    publisher={Macmillan}
}

@article{osgood:1952nature,
    title={The nature and measurement of meaning.},
    author={Osgood, Charles E},
    journal={Psychological bulletin},
    volume={49},
    number={3},
    pages={197},
    year={1952},
    publisher={American Psychological Association}
}

@article{baroni-lenci:2010distributional,
    title={Distributional memory: A general framework for corpus-based semantics},
    author={Baroni, Marco and Lenci, Alessandro},
    journal={Computational Linguistics},
    volume={36},
    number={4},
    pages={673--721},
    year={2010},
    publisher={MIT Press}
}

@article{pado-lapata:2007dependency,
    title={Dependency-based construction of semantic space models},
    author={Pad{\'o}, Sebastian and Lapata, Mirella},
    journal={Computational Linguistics},
    volume={33},
    number={2},
    pages={161--199},
    year={2007},
    publisher={MIT Press}
}

@article{lenci:2018distributional,
    title={Distributional models of word meaning},
    author={Lenci, Alessandro},
    journal={Annual review of Linguistics},
    volume={4},
    pages={151--171},
    year={2018},
    publisher={Annual Reviews}
}

@inproceedings{clark-etal:2008compositional,
    title={A compositional distributional model of meaning},
    author={Clark, Stephen and Coecke, Bob and Sadrzadeh, Mehrnoosh},
    booktitle={Proceedings of the Second Quantum Interaction Symposium (QI-2008)},
    pages={133--140},
    year={2008},
    organization={Oxford}
}

@article{mitchell-lapata:2010composition,
    title={Composition in distributional models of semantics},
    author={Mitchell, Jeff and Lapata, Mirella},
    journal={Cognitive science},
    volume={34},
    number={8},
    pages={1388--1429},
    year={2010},
    publisher={Wiley Online Library}
}

@inproceedings{blacoe-lapata:2012comparison,
    title={A comparison of vector-based representations for semantic composition},
    author={Blacoe, William and Lapata, Mirella},
    booktitle={Proceedings of the 2012 joint conference on empirical methods in natural language processing and computational natural language learning},
    pages={546--556},
    year={2012}
}

@book{markman:2013knowledge,
    title={Knowledge representation},
    author={Markman, Arthur B},
    year={2013},
    publisher={Psychology Press}
}

@inproceedings{levy-goldberg:2014dependency,
    title={Dependency-based word embeddings},
    author={Levy, Omer and Goldberg, Yoav},
    booktitle={Proceedings of the 52nd Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)},
    pages={302--308},
    year={2014}
}

@inproceedings{cotterell-schutze:2015morphological,
    title={Morphological Word-Embeddings},
    author={Cotterell, Ryan and Sch{\"u}tze, Hinrich},
    booktitle={Proceedings of the 2015 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
    pages={1287--1292},
    year={2015}
}

@inproceedings{tissier-etal:2017dict2vec,
    title={Dict2vec: Learning word embeddings using lexical dictionaries},
    author={Tissier, Julien and Gravier, Christophe and Habrard, Amaury},
    booktitle={Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing},
    pages={254--263},
    year={2017}
}

@inproceedings{klementiev-etal:2012inducing,
    title={Inducing crosslingual distributed representations of words},
    author={Klementiev, Alexandre and Titov, Ivan and Bhattarai, Binod},
    booktitle={Proceedings of COLING 2012},
    pages={1459--1474},
    year={2012}
}

@article{mikolov-etal:2013exploiting,
    title={Exploiting similarities among languages for machine translation},
    author={Mikolov, Tomas and Le, Quoc V and Sutskever, Ilya},
    journal={arXiv preprint arXiv:1309.4168},
    year={2013}
}

@inproceedings{ammar-etal:2016massively,
    title={Massively multilingual word embeddings},
    author={Ammar, Waleed and Mulcaire, George and Tsvetkov, Yulia and Lample, Guillaume and Dyer, Chris and Smith, Noah A},
    booktitle = {Proceedings of the 8th International Conference on Learning Representations (ICLR)},
    year      = {2020}
}

@inproceedings{smith-etal:2017offline,
    title={Offline bilingual word vectors, orthogonal transformations and the inverted softmax},
    author={Smith, Samuel L and Turban, David HP and Hamblin, Steven and Hammerla, Nils Y},
    booktitle = {Proceedings of the 5th International Conference on Learning Representations (ICLR)},
    year = {2017}
}

@inproceedings{artetxe-etal:2017learning,
    title={Learning bilingual word embeddings with (almost) no bilingual data},
    author={Artetxe, Mikel and Labaka, Gorka and Agirre, Eneko},
    booktitle={Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={451--462},
    year={2017}
}

@inproceedings{chen-cardie:2018unsupervised,
    title={Unsupervised Multilingual Word Embeddings},
    author={Chen, Xilun and Cardie, Claire},
    booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
    pages={261--270},
    year={2018}
}

@inproceedings{le-Mikolov:2014distributed,
    title={Distributed representations of sentences and documents},
    author={Le, Quoc and Mikolov, Tomas},
    booktitle={International conference on machine learning},
    pages={1188--1196},
    year={2014},
    organization={PMLR}
}

@inproceedings{kalchbrenner-etal:2014convolutional,
    title={A Convolutional Neural Network for Modelling Sentences},
    author={Kalchbrenner, Nal and Grefenstette, Edward and Blunsom, Phil},
    booktitle={Proceedings of the 52nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={655--665},
    year={2014}
}

@article{kiros-etal:2015skip,
    title={Skip-thought vectors},
    author={Kiros, Ryan and Zhu, Yukun and Salakhutdinov, Russ R and Zemel, Richard and Urtasun, Raquel and Torralba, Antonio and Fidler, Sanja},
    journal={Advances in neural information processing systems},
    volume={28},
    year={2015}
}

@inproceedings{hill-etal:2016learning,
    title={Learning Distributed Representations of Sentences from Unlabelled Data},
    author={Hill, Felix and Cho, Kyunghyun and Korhonen, Anna},
    booktitle={Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
    pages={1367--1377},
    year={2016}
}

@inproceedings{arora-etal:2017simple,
    title={A simple but tough-to-beat baseline for sentence embeddings},
    author={Arora, Sanjeev and Liang, Yingyu and Ma, Tengyu},
    booktitle={International conference on learning representations},
    year={2017}
}

@inproceedings{lin-etal:2017structured,
    title={A structured self-attentive sentence embedding},
    author={Lin, Zhouhan and Feng, Minwei and Santos, Cicero Nogueira dos and Yu, Mo and Xiang, Bing and Zhou, Bowen and Bengio, Yoshua},
    booktitle = {Proceedings of the 5th International Conference on Learning Representations (ICLR)},
    year = {2017}
}

@inproceedings{conneau2017supervised,
    title={Supervised Learning of Universal Sentence Representations from Natural Language Inference Data},
    author={Conneau, Alexis and Kiela, Douwe and Schwenk, Holger and Barrault, Lo{\"\i}c and Bordes, Antoine},
    booktitle={Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing},
    pages={670--680},
    year={2017}
}

@inproceedings{socher-etal:2011parsing,
    title={Parsing natural scenes and natural language with recursive neural networks},
    author={Socher, Richard and Lin, Cliff C and Manning, Chris and Ng, Andrew Y},
    booktitle={Proceedings of the 28th international conference on machine learning (ICML-11)},
    pages={129--136},
    year={2011}
}

@inproceedings{perozzi-etal:2014deepwalk,
    title={Deepwalk: Online learning of social representations},
    author={Perozzi, Bryan and Al-Rfou, Rami and Skiena, Steven},
    booktitle={Proceedings of the 20th ACM SIGKDD international conference on Knowledge discovery and data mining},
    pages={701--710},
    year={2014}
}

@inproceedings{tai-etal:2015improved,
    title={Improved Semantic Representations From Tree-Structured Long Short-Term Memory Networks},
    author={Tai, Kai Sheng and Socher, Richard and Manning, Christopher D},
    booktitle={Proceedings of the 53rd Annual Meeting of the Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},
    pages={1556--1566},
    year={2015}
}

@inproceedings{grover-leskovec:2016node2vec,
    title={node2vec: Scalable feature learning for networks},
    author={Grover, Aditya and Leskovec, Jure},
    booktitle={Proceedings of the 22nd ACM SIGKDD international conference on Knowledge discovery and data mining},
    pages={855--864},
    year={2016}
}

@article{mccann-etal:2017learned,
    title={Learned in translation: Contextualized word vectors},
    author={McCann, Bryan and Bradbury, James and Xiong, Caiming and Socher, Richard},
    journal={Advances in neural information processing systems},
    volume={30},
    year={2017}
}

@inproceedings{peters-etal:2018deep,
    title = "Deep Contextualized Word Representations",
    author = "Peters, Matthew E.  and Neumann, Mark  and Iyyer, Mohit  and Gardner, Matt  and Clark, Christopher  and Lee, Kenton  and Zettlemoyer, Luke",
    booktitle = "Proceedings of the 2018 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)",
    year = "2018"
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CHAPTER 5

@book{ross:1924aristotle,
    title={Aristotle's metaphysics},
    author={Ross, William David},
    year={1924},
    publisher={Clarendon Press}
}

@book{hamilton:1994time,
    title={Time Series Analysis},
    author={Hamilton, James Douglas},
    year={1994},
    publisher={Princeton University Press}
}

@book{chatfield:2003analysis,
    title={The analysis of time series: an introduction},
    author={Chatfield, Chris},
    year={2003},
    publisher={Chapman and hall/CRC}
}

@book{fuller:2009introduction,
    title={Introduction to statistical time series},
    author={Fuller, Wayne A},
    year={2009},
    publisher={John Wiley \& Sons}
}

@inproceedings{mikolov-etal:2010recurrent,
    title={Recurrent neural network based language model},
    author={Mikolov, Tomas and Karafi{\'a}t, Martin and Burget, Lukas and Cernock{\`y}, Jan and Khudanpur, Sanjeev},
    booktitle={Proceedings of Interspeech},
    pages={1045--1048},
    year={2010}
}

@article{werbos:1990backpropagation,
    title={Backpropagation through time: what it does and how to do it},
    author={Werbos, Paul J},
    journal={Proceedings of the IEEE},
    volume={78},
    number={10},
    pages={1550--1560},
    year={1990},
    publisher={IEEE}
}

@article{williams-peng:1990efficient,
    title={An efficient gradient-based algorithm for on-line training of recurrent network trajectories},
    author={Williams, Ronald J and Peng, Jing},
    journal={Neural computation},
    volume={2},
    number={4},
    pages={490--501},
    year={1990},
    publisher={MIT Press}
}

@book{sutskever:2013training,
    title={Training recurrent neural networks},
    author={Sutskever, Ilya},
    year={2013},
    publisher={University of Toronto Toronto}
}

@article{jaeger-haas:2004harnessing,
    title={Harnessing nonlinearity: Predicting chaotic systems and saving energy in wireless communication},
    author={Jaeger, Herbert and Haas, Harald},
    journal={science},
    volume={304},
    number={5667},
    pages={78--80},
    year={2004},
    publisher={American Association for the Advancement of Science}
}

@book{box-etal:2015time,
    title={Time series analysis: forecasting and control (4th ed.)},
    author={Box, George EP and Jenkins, Gwilym M and Reinsel, Gregory C and Ljung, Greta M},
    year={2015},
    publisher={John Wiley \& Sons}
}

@article{wu-etal:2016google,
    title={Google's neural machine translation system: Bridging the gap between human and machine translation},
    author={Wu, Yonghui and Schuster, Mike and Chen, Zhifeng and Le, Quoc V. and Norouzi, Mohammad and Macherey, Wolfgang and Krikun, Maxim and Cao, Yuan and Gao, Qin and Macherey, Klaus and Klingner, Jeff and Shah, Apurva and Johnson, Melvin and Liu, Xiaobing and Kaiser, {\L}ukasz and Gouws, Stephan and Kato, Yoshikiyo and Kudo, Taku and Kazawa, Hideto and Stevens, Keith and Kurian, George and Patil, Nishant and Wang, Wei and Young, Cliff and Smith, Jason and Riesa, Jason and Rudnick, Alex and Vinyals, Oriol and Corrado, Greg and Hughes, Macduff and Dean, Jeffrey},
    journal={arXiv preprint arXiv:1609.08144},
    year={2016}
}

@inproceedings{tang-etal:2015document,
    title={Document modeling with gated recurrent neural network for sentiment classification},
    author={Tang, Duyu and Qin, Bing and Liu, Ting},
    booktitle={Proceedings of the 2015 conference on empirical methods in natural language processing},
    pages={1422--1432},
    year={2015}
}

@inproceedings{graves-etal:2013hybrid,
    title={Hybrid speech recognition with deep bidirectional LSTM},
    author={Graves, Alex and Jaitly, Navdeep and Mohamed, Abdel-rahman},
    booktitle={IEEE workshop on automatic speech recognition and understanding},
    pages={273--278},
    year={2013},
    organization={IEEE}
}

@inproceedings{graves-et-al:2013speech,
    title={Speech recognition with deep recurrent neural networks},
    author={Graves, Alex and Mohamed, Abdel-rahman and Hinton, Geoffrey},
    booktitle={IEEE international conference on acoustics, speech and signal processing},
    pages={6645--6649},
    year={2013},
    organization={IEEE}
}

@article{huang-etal:2015bidirectional,
    title={Bidirectional LSTM-CRF models for sequence tagging},
    author={Huang, Zhiheng and Xu, Wei and Yu, Kai},
    journal={arXiv preprint arXiv:1508.01991},
    year={2015}
}

@incollection{atkinson-shiffrin:1968human,
    title={Human memory: A proposed system and its control processes},
    author={Atkinson, Richard C and Shiffrin, Richard M},
    booktitle={Psychology of learning and motivation},
    volume={2},
    pages={89--195},
    year={1968},
    publisher={Elsevier}
}

@inproceedings{weston-etal:2015memory,
    title={Memory networks},
    author={Weston, Jason and Chopra, Sumit and Bordes, Antoine},
    booktitle={Proceedings of the 3rd International Conference on Learning Representations, ICLR 2015},
    year={2015}
}

@inproceedings{khandelwal-etal:2019generalization,
    title={Generalization through Memorization: Nearest Neighbor Language Models},
    author={Khandelwal, Urvashi and Levy, Omer and Jurafsky, Dan and Zettlemoyer, Luke and Lewis, Mike},
    booktitle={Proceedings of International Conference on Learning Representations (ICLR)},
    year={2019}
}

@article{sukhbaatar-etal:2015end,
    title={End-to-end memory networks},
    author={Sukhbaatar, Sainbayar and Szlam, Arthur and Weston, Jason and Fergus, Rob},
    journal={Advances in neural information processing systems},
    volume={28},
    year={2015}
}

@inproceedings{kumar-etal:2016ask,
    title={Ask me anything: Dynamic memory networks for natural language processing},
    author={Kumar, Ankit and Irsoy, Ozan and Ondruska, Peter and Iyyer, Mohit and Bradbury, James and Gulrajani, Ishaan and Zhong, Victor and Paulus, Romain and Socher, Richard},
    booktitle={International conference on machine learning},
    pages={1378--1387},
    year={2016}
}

@article{graves-etal:2016hybrid,
    title={Hybrid computing using a neural network with dynamic external memory},
    author={Alex Graves and Greg Wayne and Malcolm Reynolds and Tim Harley and Ivo Danihelka and Agnieszka Grabska-Barwinska and Sergio Gomez Colmenarejo and Edward Grefenstette and Tiago Ramalho and John Agapiou and Adri\'{a} Puigdom\'{e}nech Badia and Karl Moritz Hermann and Yori Zwols and Georg Ostrovski and Adam Cain and Helen King and Christopher Summerfield and Phil Blunsom and Koray Kavukcuoglu and Demis Hassabis},
    journal={Nature},
    volume={538},
    number={7626},
    pages={471--476},
    year={2016},
    publisher={Nature Publishing Group}
}

@inproceedings{miller-etal:2016key,
    title={Key-Value Memory Networks for Directly Reading Documents},
    author={Miller, Alexander and Fisch, Adam and Dodge, Jesse and Karimi, Amir-Hossein and Bordes, Antoine and Weston, Jason},
    booktitle={Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing},
    pages={1400--1409},
    year={2016}
}

@article{bengio-etal:1994learning,
    title={Learning long-term dependencies with gradient descent is difficult},
    author={Bengio, Yoshua and Simard, Patrice and Frasconi, Paolo},
    journal={IEEE transactions on neural networks},
    volume={5},
    number={2},
    pages={157--166},
    year={1994},
    publisher={IEEE}
}

@article{gers-etal:2000learning,
    title={Learning to forget: Continual prediction with LSTM},
    author={Gers, Felix A and Schmidhuber, J{\"u}rgen and Cummins, Fred},
    journal={Neural computation},
    volume={12},
    number={10},
    pages={2451--2471},
    year={2000},
    publisher={MIT Press}
}

@inproceedings{chung-etal:2014empirical,
    title={Empirical evaluation of gated recurrent neural networks on sequence modeling},
    author={Chung, Junyoung and Gulcehre, Caglar and Cho, Kyunghyun and Bengio, Yoshua},
    booktitle={Proceedings of NIPS 2014 Workshop on Deep Learning, December 2014},
    year={2014}
}

@article{lopez:2008statistical,
    title={Statistical machine translation},
    author={Lopez, Adam},
    journal={ACM Computing Surveys (CSUR)},
    volume={40},
    number={3},
    pages={1--49},
    year={2008},
    publisher={ACM New York, NY, USA}
}

@inproceedings{gehring-etal:2017convencoder,
    title={A Convolutional Encoder Model for Neural Machine Translation},
    author={Gehring, Jonas and Auli, Michael and Grangier, David and Dauphin, Yann},
    booktitle={Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={123--135},
    year={2017}
}

@inproceedings{gehring-etal:2017convolutional,
    title={Convolutional sequence to sequence learning},
    author={Gehring, Jonas and Auli, Michael and Grangier, David and Yarats, Denis and Dauphin, Yann N},
    booktitle={International conference on machine learning},
    pages={1243--1252},
    year={2017},
    organization={PMLR}
}

@article{dufter-etal:2022position,
    title = "Position Information in Transformers: An Overview",
    author = {Dufter, Philipp  and
      Schmitt, Martin  and
      Sch{\"u}tze, Hinrich},
    journal = "Computational Linguistics",
    volume = "48",
    number = "3",
    year = "2022",
    address = "Cambridge, MA",
    publisher = "MIT Press",
    pages = "733--763",}

@article{dehghani-etal:2018universal,
    title={Universal transformers},
    author={Dehghani, Mostafa and Gouws, Stephan and Vinyals, Oriol and Uszkoreit, Jakob and Kaiser, {\L}ukasz},
    journal={arXiv preprint arXiv:1807.03819},
    year={2018}
}

@article{likhomanenko-etal:2021cape,
    title={CAPE: Encoding relative positions with continuous augmented positional embeddings},
    author={Likhomanenko, Tatiana and Xu, Qiantong and Synnaeve, Gabriel and Collobert, Ronan and Rogozhnikov, Alex},
    journal={Advances in Neural Information Processing Systems},
    volume={34},
    pages={16079--16092},
    year={2021}
}

@article{su-etal:2021roformer,
    title={Roformer: Enhanced transformer with rotary position embedding},
    author={Su, Jianlin and Lu, Yu and Pan, Shengfeng and Wen, Bo and Liu, Yunfeng},
    journal={arXiv preprint arXiv:2104.09864},
    year={2021}
}

@inproceedings{shaw-etal:2018self,
    title={Self-Attention with Relative Position Representations},
    author={Shaw, Peter and Uszkoreit, Jakob and Vaswani, Ashish},
    booktitle={Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 2 (Short Papers)},
    pages={464--468},
    year={2018}
}

@misc{kernes:2021master,
    Author = {Jonathan Kernes},
    url = {https://towardsdatascience.com/master-positional-encoding-part-i-63c05d90a0c3},
    Month = {05},
    Title = {Master Positional Encoding: Part I},
    Year = {2021}
}

@inproceedings{kim:2014convolutional,
    title = "Convolutional Neural Networks for Sentence Classification",
    author = "Kim, Yoon",
    booktitle = "Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing ({EMNLP})",
    month = oct,
    year = "2014",
    pages = "1746--1751",
}

@inproceedings{conneau-etal:2017very,
    title={Very Deep Convolutional Networks for Text Classification},
    author={Conneau, Alexis and Schwenk, Holger and Barrault, Lo{\"\i}c and Lecun, Yann},
    booktitle={Proceedings of the 15th Conference of the European Chapter of the Association for Computational Linguistics: Volume 1, Long Papers},
    pages={1107--1116},
    year={2017}
}

@article{zhang-etal:2015character,
    title={Character-level convolutional networks for text classification},
    author={Zhang, Xiang and Zhao, Junbo and LeCun, Yann},
    journal={Advances in neural information processing systems},
    volume={28},
    year={2015}
}

@inproceedings{dos-santos-gatti:2014deep,
    title = "Deep Convolutional Neural Networks for Sentiment Analysis of Short Texts",
    author = "Santos, C{\'\i}cero dos and Gatti, Ma{\'\i}ra",
    booktitle = "Proceedings of {COLING} 2014, the 25th International Conference on Computational Linguistics: Technical Papers",
    year = "2014",
    pages = "69--78",
}

@inproceedings{lai-etal:2015recurrent,
    title={Recurrent convolutional neural networks for text classification},
    author={Lai, Siwei and Xu, Liheng and Liu, Kang and Zhao, Jun},
    booktitle={Twenty-ninth AAAI conference on artificial intelligence},
    year={2015}
}

@article{reddy:1976speech,
    title={Speech recognition by machine: A review},
    author={Reddy, D Raj},
    journal={Proceedings of the IEEE},
    volume={64},
    number={4},
    pages={501--531},
    year={1976},
    publisher={IEEE}
}

@book{rabiner-juang:1993fundamentals,
    title={Fundamentals of speech recognition},
    author={Rabiner, Lawrence and Juang, Biing-Hwang},
    year={1993},
    publisher={Prentice-Hall, Inc.}
}

@article{davis-etal:1980comparison,
    title={Comparison of parametric representations for monosyllabic word recognition in continuously spoken sentences},
    author={Davis, Steven and Mermelstein, Paul},
    journal={IEEE transactions on acoustics, speech, and signal processing},
    volume={28},
    number={4},
    pages={357--366},
    year={1980},
    publisher={IEEE}
}

@article{picone:1993signal,
    title={Signal modeling techniques in speech recognition},
    author={Picone, Joseph W},
    journal={Proceedings of the IEEE},
    volume={81},
    number={9},
    pages={1215--1247},
    year={1993},
    publisher={IEEE}
}

@article{campbell:1997speaker,
    title={Speaker recognition: A tutorial},
    author={Campbell, Joseph P},
    journal={Proceedings of the IEEE},
    volume={85},
    number={9},
    pages={1437--1462},
    year={1997},
    publisher={IEEE}
}

@article{chorowski-etal:2019unsupervised,
    title={Unsupervised speech representation learning using wavenet autoencoders},
    author={Chorowski, Jan and Weiss, Ron J and Bengio, Samy and Van Den Oord, A{\"a}ron},
    journal={IEEE/ACM transactions on audio, speech, and language processing},
    volume={27},
    number={12},
    pages={2041--2053},
    year={2019},
    publisher={IEEE}
}

@inproceedings{graves-jaitly:2014towards,
    title={Towards end-to-end speech recognition with recurrent neural networks},
    author={Graves, Alex and Jaitly, Navdeep},
    booktitle={Proceedings of International conference on machine learning},
    pages={1764--1772},
    year={2014}
}

@inproceedings{graves-etal:2006connectionist,
    title={Connectionist temporal classification: labelling unsegmented sequence data with recurrent neural networks},
    author={Graves, Alex and Fern{\'a}ndez, Santiago and Gomez, Faustino and Schmidhuber, J{\"u}rgen},
    booktitle={Proceedings of the 23rd international conference on Machine learning},
    pages={369--376},
    year={2006}
}

@article{sutton-etal:2012introduction,
    title={An introduction to conditional random fields},
    author={Charles Sutton and Andrew McCallum},
    journal={Foundations and Trends{\textregistered} in Machine Learning},
    volume={4},
    number={4},
    pages={267--373},
    year={2012},
    publisher={Now Publishers, Inc.}
}

@inproceedings{lample-etal:2016neural,
    title={Neural Architectures for Named Entity Recognition},
    author={Lample, Guillaume and Ballesteros, Miguel and Subramanian, Sandeep and Kawakami, Kazuya and Dyer, Chris},
    booktitle={Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
    pages={260--270},
    year={2016}
}

@inproceedings{ma-hovy:2016end,
    title={End-to-end Sequence Labeling via Bi-directional LSTM-CNNs-CRF},
    author={Ma, Xuezhe and Hovy, Eduard},
    booktitle={Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={1064--1074},
    year={2016}
}

@article{li-etal:2020survey,
    title={A Survey on Deep Learning for Named Entity Recognition},
    author={Li, Jing and Sun, Aixin and Han, Jianglei and Li, Chenliang},
    journal={IEEE Transactions on Knowledge and Data Engineering},
    volume={34},
    number={1},
    pages={50--70},
    year={2020},
    publisher={IEEE}
}

@article{bojanowski-etal:2017enriching,
    title={Enriching word vectors with subword information},
    author={Bojanowski, Piotr and Grave, Edouard and Joulin, Armand and Mikolov, Tomas},
    journal={Transactions of the association for computational linguistics},
    volume={5},
    pages={135--146},
    year={2017},
    publisher={MIT Press}
}

@inproceedings{kim-etal:2016character,
    title={Character-aware neural language models},
    author={Kim, Yoon and Jernite, Yacine and Sontag, David and Rush, Alexander M},
    booktitle={Proceedings of the Thirtieth AAAI conference on artificial intelligence},
    year={2016}
}

@article{markov:1913essai,
    title={Essai d'une recherche statistique sur le texte du roman},
    author={Markov, AA},
    journal={Eugene Onegin'' illustrant la liaison des epreuve en chain (`Example of a statistical investigation of the text of ``Eugene Onegin'' illustrating the dependence between samples in chain''). In: Izvistia Imperatorskoi Akademii Nauk (Bulletin de l'Acad\'emie Imp\'eriale des Sciences de St.-P\'etersbourg). 6th ser},
    volume={7},
    pages={153--162},
    year={1913}
}

@article{shannon:1948a,
    author={Shannon, C. E.},
    journal={The Bell System Technical Journal},
    title={A mathematical theory of communication},
    year={1948},
    volume={27},
    number={3},
    pages={379-423}
}

@article{baum-petrie:1966statistical,
    title={Statistical inference for probabilistic functions of finite state Markov chains},
    author={Baum, Leonard E and Petrie, Ted},
    journal={The annals of mathematical statistics},
    volume={37},
    number={6},
    pages={1554--1563},
    year={1966},
    publisher={JSTOR}
}

@article{baum-etal:1970maximization,
    title={A maximization technique occurring in the statistical analysis of probabilistic functions of Markov chains},
    author={Baum, Leonard E and Petrie, Ted and Soules, George and Weiss, Norman},
    journal={The annals of mathematical statistics},
    volume={41},
    number={1},
    pages={164--171},
    year={1970},
    publisher={JSTOR}
}

@book{bourlard-morgan:1993connectionist,
    author = {Bourlard, Herve A. and Morgan, Nelson},
    title = {Connectionist Speech Recognition: A Hybrid Approach},
    year = {1993},
    publisher = {Kluwer Academic Publishers},
    address = {USA}
}

@article{bourlard-wellekens:1990links,
    title={Links between Markov models and multilayer perceptrons},
    author={Bourlard, Herve and Wellekens, Christian J},
    journal={IEEE Transactions on pattern analysis and machine intelligence},
    volume={12},
    number={12},
    pages={1167--1178},
    year={1990},
    publisher={IEEE}
}

@article{trentin-gori:2001survey,
    title={A survey of hybrid ANN/HMM models for automatic speech recognition},
    author={Trentin, Edmondo and Gori, Marco},
    journal={Neurocomputing},
    volume={37},
    number={1-4},
    pages={91--126},
    year={2001},
    publisher={Elsevier}
}

@article{lippmann:1989review,
    title={Review of neural networks for speech recognition},
    author={Lippmann, Richard P},
    journal={Neural computation},
    volume={1},
    number={1},
    pages={1--38},
    year={1989},
    publisher={MIT press}
}

@article{tank-hopfield:1987neural,
    title={Neural computation by concentrating information in time.},
    author={Tank, David W and Hopfield, JJ},
    journal={Proceedings of the National Academy of Sciences},
    volume={84},
    number={7},
    pages={1896--1900},
    year={1987},
    publisher={National Acad Sciences}
}

@phdthesis{bengio:1991artificial,
    author  = {Bengio, Yoshua},
    title   = {Artificial neural networks and their application to sequence recognition},
    school  = {McGill University},
    year    = {1991}
}

@article{lang-etal:1990time,
    title={A time-delay neural network architecture for isolated word recognition},
    author={Lang, Kevin J and Waibel, Alex H and Hinton, Geoffrey E},
    journal={Neural networks},
    volume={3},
    number={1},
    pages={23--43},
    year={1990},
    publisher={Elsevier}
}

@article{lecun-Bengio:1995convolutional,
    title={Convolutional networks for images, speech, and time series},
    author={LeCun, Yann and Bengio, Yoshua},
    journal={The handbook of brain theory and neural networks},
    volume={3361},
    number={10},
    pages={1995},
    year={1995},
    publisher={Cambridge, MA USA}
}

@article{hannun-etal:2014deep,
    title={Deep speech: Scaling up end-to-end speech recognition},
    author={Hannun, Awni and Case, Carl and Casper, Jared and Catanzaro, Bryan and Diamos, Greg and Elsen, Erich and Prenger, Ryan and Satheesh, Sanjeev and Sengupta, Shubho and Coates, Adam},
    journal={arXiv preprint arXiv:1412.5567},
    year={2014}
}

@inproceedings{conneau-etal:2018you,
    title={What you can cram into a single vector: Probing sentence embeddings for linguistic properties},
    author={Conneau, Alexis and Kruszewski, Germ{\'a}n and Lample, Guillaume and Barrault, Lo{\"\i}c and Baroni, Marco},
    booktitle={Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={2126--2136},
    year={2018}
}

@article{krizhevsky-etal:2017imagenet,
    title={Imagenet classification with deep convolutional neural networks},
    author={Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
    journal={Communications of the ACM},
    volume={60},
    number={6},
    pages={84--90},
    year={2017},
    publisher={AcM New York, NY, USA}
}

@article{lipton-etal:2015critical,
    title={A critical review of recurrent neural networks for sequence learning},
    author={Lipton, Zachary C and Berkowitz, John and Elkan, Charles},
    journal={arXiv preprint arXiv:1506.00019},
    year={2015}
}

@article{li-etal:2021survey,
    title={A survey of convolutional neural networks: analysis, applications, and prospects},
    author={Li, Zewen and Liu, Fan and Yang, Wenjie and Peng, Shouheng and Zhou, Jun},
    journal={IEEE transactions on neural networks and learning systems},
    year={2021},
    publisher={IEEE}
}

@article{khan-etal:2020survey,
    title={A survey of the recent architectures of deep convolutional neural networks},
    author={Khan, Asifullah and Sohail, Anabia and Zahoora, Umme and Qureshi, Aqsa Saeed},
    journal={Artificial intelligence review},
    volume={53},
    number={8},
    pages={5455--5516},
    year={2020},
    publisher={Springer}
}

@inproceedings{sundermeyer-etal:2012lstm,
    title={LSTM neural networks for language modeling},
    author={Sundermeyer, Martin and Schl{\"u}ter, Ralf and Ney, Hermann},
    booktitle={Proceedings of the Thirteenth annual conference of the international speech communication association},
    year={2012}
}

@article{oord-etal:2018representation,
    title={Representation learning with contrastive predictive coding},
    author={Oord, Aaron van den and Li, Yazhe and Vinyals, Oriol},
    journal={arXiv preprint arXiv:1807.03748},
    year={2018}
}

@article{chen-etal:2022wavlm,
    title={Wavlm: Large-scale self-supervised pre-training for full stack speech processing},
    author={Chen, Sanyuan and Wang, Chengyi and Chen, Zhengyang and Wu, Yu and Liu, Shujie and Chen, Zhuo and Li, Jinyu and Kanda, Naoyuki and Yoshioka, Takuya and Xiao, Xiong and Long Zhou and Shuo Ren and Yanmin Qian and Yao Qian and Jian Wu and Michael Zeng and Furu Wei},
    journal={IEEE Journal of Selected Topics in Signal Processing},
    volume={16},
    number={6},
    pages={1505--1518},
    year={2022},
    publisher={IEEE}
}

@inproceedings{chen-he:2021exploring,
    title={Exploring simple siamese representation learning},
    author={Chen, Xinlei and He, Kaiming},
    booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
    pages={15750--15758},
    year={2021}
}

@inproceedings{bao-etal:2021beit,
    title={BEiT: BERT Pre-Training of Image Transformers},
    author={Bao, Hangbo and Dong, Li and Piao, Songhao and Wei, Furu},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2021}
}

@inproceedings{he-etal:2022masked,
    title={Masked autoencoders are scalable vision learners},
    author={He, Kaiming and Chen, Xinlei and Xie, Saining and Li, Yanghao and Doll{\'a}r, Piotr and Girshick, Ross},
    booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
    pages={16000--16009},
    year={2022}
}

@article{li-etal:2021align,
    title={Align before fuse: Vision and language representation learning with momentum distillation},
    author={Li, Junnan and Selvaraju, Ramprasaath and Gotmare, Akhilesh and Joty, Shafiq and Xiong, Caiming and Hoi, Steven Chu Hong},
    journal={Advances in neural information processing systems},
    volume={34},
    pages={9694--9705},
    year={2021}
}

@inproceedings{kim-etal:2021vilt,
    title={Vilt: Vision-and-language transformer without convolution or region supervision},
    author={Kim, Wonjae and Son, Bokyung and Kim, Ildoo},
    booktitle={Proceedings of International Conference on Machine Learning},
    pages={5583--5594},
    year={2021},
    organization={PMLR}
}

@inproceedings{chuang-etal:2020speechbert,
    title={SpeechBERT: An Audio-and-Text Jointly Learned Language Model for End-to-End Spoken Question Answering},
    author={Chuang, Yung-Sung and Liu, Chi-Liang and Lee, Hung-yi and Lee, Lin-shan},
    booktitle={Proceedings of Interspeech 2020},
    pages={4168--4172},
    year={2020}
}

@inproceedings{conneau-etal:2017supervised,
    title={Supervised Learning of Universal Sentence Representations from Natural Language Inference Data},
    author={Conneau, Alexis and Kiela, Douwe and Schwenk, Holger and Barrault, Lo{\"\i}c and Bordes, Antoine},
    booktitle={Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing},
    pages={670--680},
    year={2017}
}

@article{cer-etal:2018universal,
    title={Universal sentence encoder},
    author={Daniel Cer and Yinfei Yang and Sheng-yi Kong and Nan Hua and Nicole Limtiaco and Rhomni St. John and Noah Constant and Mario Guajardo-Cespedes and Steve Yuan and Chris Tar and Yun-Hsuan Sung and Brian Strope and Ray Kurzweil},
    journal={arXiv preprint arXiv:1803.11175},
    year={2018}
}

@inproceedings{yang-etal:2016hierarchical,
  title={Hierarchical attention networks for document classification},
  author={Yang, Zichao and Yang, Diyi and Dyer, Chris and He, Xiaodong and Smola, Alex and Hovy, Eduard},
  booktitle={Proceedings of the 2016 conference of the North American chapter of the association for computational linguistics: human language technologies},
  pages={1480--1489},
  year={2016}
}

@article{vinyals-etal:2015grammar,
    title={Grammar as a foreign language},
    author={Vinyals, Oriol and Kaiser, {\L}ukasz and Koo, Terry and Petrov, Slav and Sutskever, Ilya and Hinton, Geoffrey},
    journal={Advances in neural information processing systems},
    volume={28},
    year={2015}
}

@inproceedings{chen-etal:2020generative,
    title={Generative pretraining from pixels},
    author={Chen, Mark and Radford, Alec and Child, Rewon and Wu, Jeffrey and Jun, Heewoo and Luan, David and Sutskever, Ilya},
    booktitle={International conference on machine learning},
    pages={1691--1703},
    year={2020},
    organization={PMLR}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CHAPTER 6

@book{chang:1967reflections,
    title={Reflections on things at hand},
    author = {Chang, Wing-Tsit},
    year={1967},
    publisher={Columbia University Press}
}

@article{sutskever-etal:2014sequence,
    title={Sequence to sequence learning with neural networks},
    author={Sutskever, Ilya and Vinyals, Oriol and Le, Quoc V},
    journal={Advances in neural information processing systems},
    volume={27},
    year={2014}
}

@book{sternberg:1996cognitive,
    title={Cognitive psychology},
    author={Sternberg, Robert J},
    year={1996},
    publisher={Harcourt Brace College Publishers}
}

@book{neisser:2014cognitive,
    title={Cognitive Psychology: Classic Edition},
    author={Neisser, Ulric},
    year={2014},
    publisher={Psychology Press}
}

@misc{ruder:2017deep,
    howpublished = {\url{https://ruder.io/deep-learning-nlp-best-practices/index.html}},
    title = {Deep Learning for NLP Best Practices},
    author = {Sebastian Ruder},
    year = {2017}
}

@article{opitz-maclin:1999popular,
    title={Popular ensemble methods: An empirical study},
    author={Opitz, David and Maclin, Richard},
    journal={Journal of artificial intelligence research},
    volume={11},
    pages={169--198},
    year={1999}
}

@book{zhou:2012ensemble,
    title={Ensemble methods: foundations and algorithms},
    author={Zhou, Zhi-Hua},
    year={2012},
    publisher={CRC press}
}

@inproceedings{maruf-etal:2019selective,
    title={Selective Attention for Context-aware Neural Machine Translation},
    author={Maruf, Sameen and Martins, Andr{\'e} FT and Haffari, Gholamreza},
    booktitle={Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)},
    pages={3092--3102},
    year={2019}
}

@inproceedings{koehn-etal:2003statistical,
    title={Statistical Phrase-Based Translation},
    author={Koehn, Philipp and Och, Franz Josef and Marcu, Daniel},
    booktitle={Proceedings of the 2003 Human Language Technology Conference of the North American Chapter of the Association for Computational Linguistics},
    pages={127--133},
    year={2003}
}

@inproceedings{chiang:2005hierarchical,
    title={A hierarchical phrase-based model for statistical machine translation},
    author={Chiang, David},
    booktitle={Proceedings of the 43rd annual meeting of the association for computational linguistics (acl'05)},
    pages={263--270},
    year={2005}
}

@inproceedings{vogel-etal:1996hmm,
    title={HMM-based word alignment in statistical translation},
    author={Vogel, Stephan and Ney, Hermann and Tillmann, Christoph},
    booktitle={COLING 1996 Volume 2: The 16th International Conference on Computational Linguistics},
    year={1996}
}

@article{och-ney:2003systematic,
    title={A Systematic Comparison of Various Statistical Alignment Models},
    author={Och, Franz Josef and Ney, Hermann},
    journal={Computational Linguistics},
    volume={29},
    number={1},
    pages={19--51},
    year={2003}
}

@inproceedings{taskar-etal:2005discriminative,
    title={A discriminative matching approach to word alignment},
    author={Taskar, Ben and Lacoste-Julien, Simon and Klein, Dan},
    booktitle={Proceedings of Human Language Technology Conference and Conference on Empirical Methods in Natural Language Processing},
    pages={73--80},
    year={2005}
}

@inproceedings{dyer-etal:2013simple,
    title={A simple, fast, and effective reparameterization of ibm model 2},
    author={Dyer, Chris and Chahuneau, Victor and Smith, Noah A},
    booktitle={Proceedings of the 2013 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
    pages={644--648},
    year={2013}
}

@inproceedings{rush-etal:2015neural,
    title={A Neural Attention Model for Abstractive Sentence Summarization},
    author={Rush, Alexander M and Chopra, Sumit and Weston, Jason},
    booktitle={Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing},
    pages={379--389},
    year={2015}
}

@inproceedings{tu-etal:2016modeling,
    title={Modeling Coverage for Neural Machine Translation},
    author={Tu, Zhaopeng and Lu, Zhengdong and Liu, Yang and Liu, Xiaohua and Li, Hang},
    booktitle={Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={76--85},
   year={2016}
}

@inproceedings{li-etal:2019word,
    title={On the word alignment from neural machine translation},
    author={Li, Xintong and Li, Guanlin and Liu, Lemao and Meng, Max and Shi, Shuming},
    booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
    pages={1293--1303},
    year={2019}
}

@inproceedings{garg-etal:2019jointly,
    title={Jointly Learning to Align and Translate with Transformer Models},
    author={Garg, Sarthak and Peitz, Stephan and Nallasamy, Udhyakumar and Paulik, Matthias},
    booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
    pages={4453--4462},
    year={2019}
}

@inproceedings{mi-etal:2016supervised,
    title={Supervised Attentions for Neural Machine Translation},
    author={Mi, Haitao and Wang, Zhiguo and Ittycheriah, Abe},
    booktitle={Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing},
    pages={2283--2288},
    year={2016}
}

@inproceedings{liu-etal:2016neural,
    title={Neural Machine Translation with Supervised Attention},
    author={Liu, Lemao and Utiyama, Masao and Finch, Andrew and Sumita, Eiichiro},
    booktitle={Proceedings of COLING 2016, the 26th International Conference on Computational Linguistics: Technical Papers},
    pages={3093--3102},
    year={2016}
}

@inproceedings{koehn:2004pharaoh,
    title={Pharaoh: a beam search decoder for phrase-based statistical machine translation models},
    author={Koehn, Philipp},
    booktitle={Conference of the Association for Machine Translation in the Americas},
    pages={115--124},
    year={2004},
    organization={Springer}
}

@inproceedings{you-etal:2020hard,
    title={Hard-Coded Gaussian Attention for Neural Machine Translation},
    author={You, Weiqiu and Sun, Simeng and Iyyer, Mohit},
    booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
    pages={7689--7700},
    year={2020}
}

@inproceedings{yang-etal:2018modeling,
    title={Modeling Localness for Self-Attention Networks},
    author={Yang, Baosong and Tu, Zhaopeng and Wong, Derek F and Meng, Fandong and Chao, Lidia S and Zhang, Tong},
    booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
    pages={4449--4458},
    year={2018}
}

@inproceedings{guo-etal:2019gaussian,
    title={Gaussian transformer: a lightweight approach for natural language inference},
    author={Guo, Maosheng and Zhang, Yu and Liu, Ting},
    booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
    pages={6489--6496},
    year={2019}
}

@inproceedings{cohn-etal:2016incorporating,
    title={Incorporating Structural Alignment Biases into an Attentional Neural Translation Model},
    author={Cohn, Trevor and Hoang, Cong Duy Vu and Vymolova, Ekaterina and Yao, Kaisheng and Dyer, Chris and Haffari, Gholamreza},
    booktitle={Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
    pages={876--885},
    year={2016}
}

@inproceedings{feng-etal:2016improving,
    title={Improving attention modeling with implicit distortion and fertility for machine translation},
    author={Feng, Shi and Liu, Shujie and Yang, Nan and Li, Mu and Zhou, Ming and Zhu, Kenny},
    booktitle={Proceedings of COLING 2016, the 26th International Conference on Computational Linguistics: Technical Papers},
    pages={3082--3092},
    year={2016}
}

@inproceedings{raffel-etal:2017online,
    title={Online and linear-time attention by enforcing monotonic alignments},
    author={Raffel, Colin and Luong, Minh-Thang and Liu, Peter J and Weiss, Ron J and Eck, Douglas},
    booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
    pages={2837--2846},
    year={2017}
}

@inproceedings{chiu-raffel:2018monotonic,
  title={Monotonic Chunkwise Attention},
  author={Chiu, Chung-Cheng and Raffel, Colin},
  booktitle = {Proceedings of the 8th International Conference on Learning Representations {ICLR}},
  year = {2018}
}

@inproceedings{chen-etal:2018syntax,
    title={Syntax-directed attention for neural machine translation},
    author={Chen, Kehai and Wang, Rui and Utiyama, Masao and Sumita, Eiichiro and Zhao, Tiejun},
    booktitle={Proceedings of the AAAI conference on artificial intelligence},
    year={2018}
}

@inproceedings{xu-etal:2021syntax,
    title={Syntax-Enhanced Pre-trained Model},
    author={Xu, Zenan and Guo, Daya and Tang, Duyu and Su, Qinliang and Shou, Linjun and Gong, Ming and Zhong, Wanjun and Quan, Xiaojun and Jiang, Daxin and Duan, Nan},
    booktitle={Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},
    pages={5412--5422},
    year={2021}
}

@inproceedings{zhang-etal:2020sg,
    title={SG-Net: Syntax-guided machine reading comprehension},
    author={Zhang, Zhuosheng and Wu, Yuwei and Zhou, Junru and Duan, Sufeng and Zhao, Hai and Wang, Rui},
    booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
    pages={9636--9643},
    year={2020}
}

@inproceedings{nguyen-etal:2020tree,
    author    = {Xuan-Phi Nguyen and Shafiq Joty and Steven Hoi and Richard Socher},
    title     = {Tree-Structured Attention with Hierarchical Accumulation},
    booktitle = {Proceedings of the 8th International Conference on Learning Representations {ICLR}},
    year      = {2020}
}

@article{lample-conneau:2019cross,
    title={Cross-lingual language model pretraining},
    author={Lample, Guillaume and Conneau, Alexis},
    journal={arXiv preprint arXiv:1901.07291},
    year={2019}
}

@article{raffel-etal:2020exploring,
    title={Exploring the limits of transfer learning with a unified text-to-text transformer.},
    author  = {Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu},
    journal={Journal of Machine Learning Research},
    volume={21},
    number={140},
    pages={1--67},
    year={2020}
}

@article{ouyang-etal:2022training,
    title={Training language models to follow instructions with human feedback},
    author = {Ouyang, Long and Wu, Jeffrey and Jiang, Xu and Almeida, Diogo and Wainwright, Carroll L. and Mishkin, Pamela and Zhang, Chong and Agarwal, Sandhini and Slama, Katarina and Ray, Alex and Schulman, John and Hilton, Jacob and Kelton, Fraser and Miller, Luke and Simens, Maddie and Askell, Amanda and Welinder, Peter and Christiano, Paul F. and Leike, Jan and Lowe, Ryan},
    journal={Advances in Neural Information Processing Systems},
    volume={35},
    pages={27730--27744},
    year={2022}
}

@article{touvron-etal:2023llama,
    title={LLaMA: Open and efficient foundation language models},
    author={Hugo Touvron and Thibaut Lavril and Gautier Izacard and Xavier Martinet and Marie-Anne Lachaux and Timoth{\'e}e Lacroix and Baptiste Rozi{\`e}re and Naman Goyal and Eric Hambro and Faisal Azhar and Aurelien Rodriguez and Armand Joulin and Edouard Grave and Guillaume Lample},
    journal={arXiv preprint arXiv:2302.13971},
    year={2023}
}

@article{touvron-etal:2023llama2,
    title={Llama 2: Open foundation and fine-tuned chat models},
    author={Hugo Touvron and Louis Martin and Kevin Stone and Peter Albert and Amjad Almahairi and Yasmine Babaei and Nikolay Bashlykov and Soumya Batra and Prajjwal Bhargava and Shruti Bhosale and Dan Bikel and Lukas Blecher and Cristian Canton Ferrer and Moya Chen and Guillem Cucurull and David Esiobu and Jude Fernandes and Jeremy Fu and Wenyin Fu and Brian Fuller and Cynthia Gao and Vedanuj Goswami and Naman Goyal and Anthony Hartshorn and Saghar Hosseini and Rui Hou and Hakan Inan and Marcin Kardas and Viktor Kerkez and Madian Khabsa and Isabel Kloumann and Artem Korenev and Punit Singh Koura and Marie-Anne Lachaux and Thibaut Lavril and Jenya Lee and Diana Liskovich and Yinghai Lu and Yuning Mao and Xavier Martinet and Todor Mihaylov and Pushkar Mishra and Igor Molybog and Yixin Nie and Andrew Poulton and Jeremy Reizenstein and Rashi Rungta and Kalyan Saladi and Alan Schelten and Ruan Silva and Eric Michael Smith and Ranjan Subramanian and Xiaoqing Ellen Tan and Binh Tang and Ross Taylor and Adina Williams and Jian Xiang Kuan and Puxin Xu and Zheng Yan and Iliyan Zarov and Yuchen Zhang and Angela Fan and Melanie Kambadur and Sharan Narang and Aurelien Rodriguez and Robert Stojnic and Sergey Edunov and Thomas Scialom},
    journal={arXiv preprint arXiv:2307.09288},
    year={2023}
}

@article{dubey2024llama,
  title={The llama 3 herd of models},
  author={Dubey, Abhimanyu and Jauhri, Abhinav and Pandey, Abhinav and Kadian, Abhishek and Al-Dahle, Ahmad and Letman, Aiesha and Mathur, Akhil and Schelten, Alan and Yang, Amy and Fan, Angela and others},
  journal={arXiv preprint arXiv:2407.21783},
  year={2024}
}

@inproceedings{chen-etal:2020uniter,
    title={Uniter: Universal image-text representation learning},
    author={Chen, Yen-Chun and Li, Linjie and Yu, Licheng and El Kholy, Ahmed and Ahmed, Faisal and Gan, Zhe and Cheng, Yu and Liu, Jingjing},
    booktitle={Proceedings of European conference on computer vision},
    pages={104--120},
    year={2020}
}

@article{lu-etal:2016hierarchical,
    title={Hierarchical question-image co-attention for visual question answering},
    author={Lu, Jiasen and Yang, Jianwei and Batra, Dhruv and Parikh, Devi},
    journal={Advances in neural information processing systems},
    volume={29},
    year={2016}
}

@inproceedings{velivckovic-etal:2018graph,
    title={Graph Attention Networks},
    author={Veli{\v{c}}kovi{\'c}, Petar and Cucurull, Guillem and Casanova, Arantxa and Romero, Adriana and Li{\`o}, Pietro and Bengio, Yoshua},
    booktitle={International Conference on Learning Representations},
    year={2018}
}

@article{lee-etal:2019attention,
    title={Attention models in graphs: A survey},
    author={Lee, John Boaz and Rossi, Ryan A and Kim, Sungchul and Ahmed, Nesreen K and Koh, Eunyee},
    journal={ACM Transactions on Knowledge Discovery from Data (TKDD)},
    volume={13},
    number={6},
    pages={1--25},
    year={2019},
    publisher={ACM New York, NY, USA}
}

@article{wiher-etal:2022decoding,
    title={On decoding strategies for neural text generators},
    author={Wiher, Gian and Meister, Clara and Cotterell, Ryan},
    journal={Transactions of the Association for Computational Linguistics},
    volume={10},
    pages={997--1012},
    year={2022},
    publisher={MIT Press}
}

@inproceedings{li-etal:2021mixup,
    title={Mixup Decoding for Diverse Machine Translation},
    author={Li, Jicheng and Gao, Pengzhi and Wu, Xuanfu and Feng, Yang and He, Zhongjun and Wu, Hua and Wang, Haifeng},
    booktitle={Findings of the Association for Computational Linguistics: EMNLP 2021},
    pages={312--320},
    year={2021}
}

@inproceedings{he-etal:2018sequence,
    title={Sequence to Sequence Mixture Model for Diverse Machine Translation},
    author={He, Xuanli and Haffari, Gholamreza and Norouzi, Mohammad},
    booktitle={Proceedings of the 22nd Conference on Computational Natural Language Learning},
    pages={583--592},
    year={2018}
}

@inproceedings{shen-etal:2019mixture,
    title={Mixture models for diverse machine translation: Tricks of the trade},
    author={Shen, Tianxiao and Ott, Myle and Auli, Michael and Ranzato, Marc'Aurelio},
    booktitle={International conference on machine learning},
    pages={5719--5728},
    year={2019}
}

@inproceedings{wu-etal:2020generating,
    title={Generating Diverse Translation from Model Distribution with Dropout},
    author={Wu, Xuanfu and Feng, Yang and Shao, Chenze},
    booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
    pages={1088--1097},
    year={2020}
}

@inproceedings{sun-etal:2020generating,
    title={Generating diverse translation by manipulating multi-head attention},
    author={Sun, Zewei and Huang, Shujian and Wei, Hao-Ran and Dai, Xin-yu and Chen, Jiajun},
    booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
    volume={34},
    pages={8976--8983},
    year={2020}
}

@inproceedings{he-etal:2016improved,
    title={Improved neural machine translation with SMT features},
    author={He, Wei and He, Zhongjun and Wu, Hua and Wang, Haifeng},
    booktitle={Proceedings of the Thirtieth AAAI conference on artificial intelligence},
    year={2016}
}

@inproceedings{huang-etal:2017finish,
    title={When to Finish? Optimal Beam Search for Neural Text Generation (modulo beam size)},
    author={Huang, Liang and Zhao, Kai and Ma, Mingbo},
    booktitle={Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing},
    pages={2134--2139},
    year={2017}
}

@inproceedings{yang-etal:2018breaking,
    title={Breaking the Beam Search Curse: A Study of (Re-) Scoring Methods and Stopping Criteria for Neural Machine Translation},
    author={Yang, Yilin and Huang, Liang and Ma, Mingbo},
    booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
    pages={3054--3059},
    year={2018}
}

@inproceedings{och:2003mert,
    title={Minimum error rate training in statistical machine translation},
    author={Och, Franz Josef},
    booktitle={Proceedings of the 41st annual meeting of the Association for Computational Linguistics},
    pages={160--167},
    year={2003}
}

@inproceedings{murray-chiang:2018correcting,
    title={Correcting Length Bias in Neural Machine Translation},
    author={Murray, Kenton and Chiang, David},
    booktitle={Proceedings of the Third Conference on Machine Translation: Research Papers},
    pages={212--223},
    year={2018}
}

@article{chiang:2007hierarchical,
    title={Hierarchical phrase-based translation},
    author={Chiang, David},
    journal={computational linguistics},
    volume={33},
    number={2},
    pages={201--228},
    year={2007},
    publisher={MIT Press}
}

@inproceedings{jean-etal:2015montreal,
    title={Montreal neural machine translation systems for WMT'15},
    author={Jean, S{\'e}bastien and Firat, Orhan and Cho, Kyunghyun and Memisevic, Roland and Bengio, Yoshua},
    booktitle={Proceedings of the tenth workshop on statistical machine translation},
    pages={134--140},
    year={2015}
}

@article{williams:1992simple,
    title={Simple statistical gradient-following algorithms for connectionist reinforcement learning},
    author={Williams, Ronald J},
    journal={Machine learning},
    volume={8},
    pages={229--256},
    year={1992},
    publisher={Springer}
}

@inproceedings{li-etal:2018simple,
    title={A simple and effective approach to coverage-aware neural machine translation},
    author={Li, Yanyang and Xiao, Tong and Li, Yinqiao and Wang, Qiang and Xu, Changming and Zhu, Jingbo},
    booktitle={Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)},
    pages={292--297},
    year={2018}
}

@article{chorowski-jaitly:2017towards,
    title={Towards Better Decoding and Language Model Integration in Sequence to Sequence Models},
    author={Chorowski, Jan and Jaitly, Navdeep},
    journal={Proc. Interspeech 2017},
    pages={523--527},
    year={2017}
}

@inproceedings{mi-etal:2016coverage,
    title={Coverage Embedding Models for Neural Machine Translation},
    author={Mi, Haitao and Sankaran, Baskaran and Wang, Zhiguo and Ittycheriah, Abe},
    booktitle={Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing},
    pages={955--960},
    year={2016}
}

@inproceedings{malaviya-etal:2018sparse,
    title={Sparse and Constrained Attention for Neural Machine Translation},
    author={Malaviya, Chaitanya and Ferreira, Pedro and Martins, Andr{\'e} FT},
    booktitle={Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)},
    pages={370--376},
    year={2018}
}

@article{sankaran-etal:2016temporal,
    title={Temporal attention model for neural machine translation},
    author={Sankaran, Baskaran and Mi, Haitao and Al-Onaizan, Yaser and Ittycheriah, Abe},
    journal={arXiv preprint arXiv:1608.02927},
    year={2016}
}

@inproceedings{see-etal:2017get,
    title={Get To The Point: Summarization with Pointer-Generator Networks},
    author={See, Abigail and Liu, Peter J and Manning, Christopher D},
    booktitle={Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={1073--1083},
    year={2017}
}

@inproceedings{gu-etal:2018non,
    title={Non-Autoregressive Neural Machine Translation},
    author={Gu, Jiatao and Bradbury, James and Xiong, Caiming and Li, Victor O.K. and Socher, Richard},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2018}
}

@inproceedings{kikuchi-etal:2016controlling,
    title={Controlling Output Length in Neural Encoder-Decoders},
    author={Kikuchi, Yuta and Neubig, Graham and Sasano, Ryohei and Takamura, Hiroya and Okumura, Manabu},
    booktitle={Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing},
    pages={1328--1338},
    year={2016}
}

@inproceedings{koehn-knowles:2017six,
    title={Six Challenges for Neural Machine Translation},
    author={Koehn, Philipp and Knowles, Rebecca},
    booktitle={Proceedings of the First Workshop on Neural Machine Translation},
    pages={28--39},
    year={2017}
}

@inproceedings{stahlberg-byrne:2019nmt,
    title={On NMT Search Errors and Model Errors: Cat Got Your Tongue?},
    author={Stahlberg, Felix and Byrne, Bill},
    booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
    pages={3356--3362},
    year={2019}
}

@inproceedings{klein-etal:2017opennmt,
    title={OpenNMT: Open-Source Toolkit for Neural Machine Translation},
    author={Klein, Guillaume and Kim, Yoon and Deng, Yuntian and Senellart, Jean and Rush, Alexander M},
    booktitle={Proceedings of ACL 2017, System Demonstrations},
    pages={67--72},
    year={2017}
}

@inproceedings{ott-etal:2019fairseq,
    title={fairseq: A Fast, Extensible Toolkit for Sequence Modeling},
    author={Ott, Myle and Edunov, Sergey and Baevski, Alexei and Fan, Angela and Gross, Sam and Ng, Nathan and Grangier, David and Auli, Michael},
    booktitle={Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics (Demonstrations)},
    pages={48--53},
    year={2019}
}

@article{casacuberta-etal:2009human,
    title={Human interaction for high-quality machine translation},
    author={Casacuberta, Francisco and Civera, Jorge and Cubel, Elsa and Lagarda, Antonio L and Lapalme, Guy and Macklovitch, Elliott and Vidal, Enrique},
    journal={Communications of the ACM},
    volume={52},
    number={10},
    pages={135--138},
    year={2009}
}

@inproceedings{grissom-ii-etal:2014dont,
    title = "Don{'}t Until the Final Verb Wait: Reinforcement Learning for Simultaneous Machine Translation",
    author = "Grissom II, Alvin  and He, He  and Boyd-Graber, Jordan  and Morgan, John  and Daum{\'e} III, Hal",
    booktitle = "Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing ({EMNLP})",
    year = "2014",
    pages = "1342--1352",
}

@inproceedings{ma-etal:2019stacl,
    title={STACL: Simultaneous Translation with Implicit Anticipation and Controllable Latency using Prefix-to-Prefix Framework},
    author={Ma, Mingbo and Huang, Liang and Xiong, Hao and Zheng, Renjie and Liu, Kaibo and Zheng, Baigong and Zhang, Chuanqiang and He, Zhongjun and Liu, Hairong and Li, Xing and Wu, Hua and Wang, Haifeng},
    booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
    pages={3025--3036},
    year={2019}
}

@article{cho-esipova:2016can,
    title={Can neural machine translation do simultaneous translation?},
    author={Cho, Kyunghyun and Esipova, Masha},
    journal={arXiv preprint arXiv:1606.02012},
    year={2016}
}

@inproceedings{gu-etal:2017learning,
    title={Learning to translate in real-time with neural machine translation},
    author={Gu, Jiatao and Neubig, Graham and Cho, Kyunghyun and Li, Victor O.K.},
    booktitle={Proceedings of the European Chapter of the Association for Computational Linguistics (EACL) Conference, 2017},
    year={2017}
}

@inproceedings{zheng-etal:2019simpler,
    title={Simpler and Faster Learning of Adaptive Policies for Simultaneous Translation},
    author={Zheng, Baigong and Zheng, Renjie and Ma, Mingbo and Huang, Liang},
    booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
    pages={1349--1354},
    year={2019}
}

@article{knight:1999decoding,
    title={Decoding complexity in word-replacement translation models},
    author={Knight, Kevin},
    journal={Computational linguistics},
    volume={25},
    number={4},
    pages={607--615},
    year={1999}
}

@article{germann-etal:2004fast,
    title={Fast and optimal decoding for machine translation},
    author={Germann, Ulrich and Jahr, Michael and Knight, Kevin and Marcu, Daniel and Yamada, Kenji},
    journal={Artificial Intelligence},
    volume={154},
    number={1-2},
    pages={127--143},
    year={2004},
    publisher={Elsevier}
}

@inproceedings{zaslavskiy-etal:2009phrase,
    title={Phrase-based statistical machine translation as a traveling salesman problem},
    author={Zaslavskiy, Mikhail and Dymetman, Marc and Cancedda, Nicola},
    booktitle={Proceedings of the Joint Conference of the 47th Annual Meeting of the ACL and the 4th International Joint Conference on Natural Language Processing of the AFNLP},
    pages={333--341},
    year={2009}
}

@inproceedings{chang-collins:2011exact,
    title={Exact Decoding of Phrase-Based Translation Models through Lagrangian Relaxation},
    author={Chang, Yin-Wen and Collins, Michael},
    booktitle={Proceedings of the 2011 Conference on Empirical Methods in Natural Language Processing},
    pages={26--37},
    year={2011}
}

@article{rush-etal:2012tutorial,
    title={A tutorial on dual decomposition and lagrangian relaxation for inference in natural language processing},
    author={Rush, Alexander M and Collins, MJ},
    journal={Journal of Artificial Intelligence Research},
    volume={45},
    pages={305--362},
    year={2012}
}

@article{de-etal:2010hierarchical,
    title={Hierarchical phrase-based translation with weighted finite-state transducers and shallow-n grammars},
    author={de Gispert, Adri{\`a} and Iglesias, Gonzalo and Blackwood, Graeme and R. Banga, Eduardo and Byrne, William},
    journal={Computational linguistics},
    volume={36},
    number={3},
    pages={505--533},
    year={2010}
}

@article{allauzen-etal:2014pushdown,
    title={Pushdown automata in statistical machine translation},
    author={Allauzen, Cyril and Byrne, Bill and de Gispert, Adri{\`a} and Iglesias, Gonzalo and Riley, Michael},
    journal={Computational Linguistics},
    volume={40},
    number={3},
    pages={687--723},
    year={2014}
}

@article{meister-etal:2020best,
    title = "Best-First Beam Search",
    author = "Meister, Clara  and Vieira, Tim  and Cotterell, Ryan",
    journal = "Transactions of the Association for Computational Linguistics",
    volume = "8",
    year = "2020",
    address = "Cambridge, MA",
    publisher = "MIT Press",
    pages = "795--809"
}

@inproceedings{hoang-etal:2017towards,
    title={Towards Decoding as Continuous Optimisation in Neural Machine Translation},
    author={Hoang, Cong Duy Vu and Haffari, Gholamreza and Cohn, Trevor},
    booktitle={Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing},
    pages={146--156},
    year={2017}
}

@article{kumar-etal:2021controlled,
    title={Controlled text generation as continuous optimization with multiple constraints},
    author={Kumar, Sachin and Malmi, Eric and Severyn, Aliaksei and Tsvetkov, Yulia},
    journal={Advances in Neural Information Processing Systems},
    volume={34},
    pages={14542--14554},
    year={2021}
}

@inproceedings{vijayakumar-etal:2018diverse,
    title={Diverse beam search for improved description of complex scenes},
    author={Vijayakumar, Ashwin and Cogswell, Michael and Selvaraju, Ramprasaath and Sun, Qing and Lee, Stefan and Crandall, David and Batra, Dhruv},
    booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
    volume={32},
    year={2018}
}

@article{li-jurafsky:2016mutual,
    title={Mutual information and diverse decoding improve neural machine translation},
    author={Li, Jiwei and Jurafsky, Dan},
    journal={arXiv preprint arXiv:1601.00372},
    year={2016}
}

@article{xiao-etal:2013bagging,
    title={Bagging and boosting statistical machine translation systems},
    author={Xiao, Tong and Zhu, Jingbo and Liu, Tongran},
    journal={Artificial Intelligence},
    volume={195},
    pages={496--527},
    year={2013},
    publisher={Elsevier}
}

@inproceedings{holtzman-etal:2020curious,
    title={The Curious Case of Neural Text Degeneration},
    author={Holtzman, Ari and Buys, Jan and Du, Li and Forbes, Maxwell and Choi, Yejin},
    booktitle = {Proceedings of the 6th International Conference on Learning Representations {ICLR}},
    year = {2020}
}

@inproceedings{boulanger-etal:2013audio,
    title={Audio Chord Recognition with Recurrent Neural Networks.},
    author={Boulanger-Lewandowski, Nicolas and Bengio, Yoshua and Vincent, Pascal},
    year={2013},
    booktitle={Proceedings of 14th International Society for Music Information Retrieval Conference}
}

@inproceedings{barrault-etal:2020findings,
    title = "Findings of the 2020 Conference on Machine Translation ({WMT}20)",
    author = {Barrault, Lo{\"i}c and Biesialska, Magdalena and Bojar, Ond{\v{r}}ej and Costa-juss{\`a}, Marta R. and Federmann, Christian and Graham, Yvette and Grundkiewicz, Roman and Haddow, Barry and Huck, Matthias and Joanis, Eric and Kocmi, Tom and Koehn, Philipp and Lo, Chi-kiu and Ljube{\v{s}}i{\'c}, Nikola and Monz, Christof and Morishita, Makoto and Nagata, Masaaki and Nakazawa, Toshiaki and Pal, Santanu and Post, Matt and Zampieri, Marcos},
    booktitle = "Proceedings of the Fifth Conference on Machine Translation",
    year = "2020",
    pages = "1--55"
}

@inproceedings{akhbardeh-etal:2021findings,
    title = "Findings of the 2021 Conference on Machine Translation ({WMT}21)",
    author="Akhbardeh, Farhad and Arkhangorodsky, Arkady and Biesialska, Magdalena and Bojar, Ond{\v{r}}ej and Chatterjee, Rajen and Chaudhary, Vishrav and Costa-jussa, Marta R. and Espa{\~n}a-Bonet, Cristina and Fan, Angela and Federmann, Christian and Freitag, Markus and Graham, Yvette and Grundkiewicz, Roman and Haddow, Barry and Harter, Leonie and Heafield, Kenneth and Homan, Christopher and Huck, Matthias and Amponsah-Kaakyire, Kwabena and Kasai, Jungo and Khashabi, Daniel and Knight, Kevin and Kocmi, Tom and Koehn, Philipp and Lourie, Nicholas and Monz, Christof and Morishita, Makoto and Nagata, Masaaki and Nagesh, Ajay and Nakazawa, Toshiaki and Negri, Matteo and Pal, Santanu and Tapo, Allahsera Auguste and Turchi, Marco and Vydrin, Valentin and Zampieri, Marcos",
    booktitle = "Proceedings of the Sixth Conference on Machine Translation",
    year = "2021",
    pages = "1--88"
}

@inproceedings{hildebrand-etal:2008combination,
    title={Combination of machine translation systems via hypothesis selection from combined n-best lists},
    author={Hildebrand, Almut Silja and Vogel, Stephan},
    booktitle={Proceedings of the 8th Conference of the Association for Machine Translation in the Americas: Student Research Workshop},
    pages={254--261},
    year={2008}
}

@inproceedings{matusov-etal:2006computing,
    title={Computing consensus translation for multiple machine translation systems using enhanced hypothesis alignment},
    author={Matusov, Evgeny and Ueffing, Nicola and Ney, Hermann},
    booktitle={11th Conference of the European Chapter of the Association for Computational Linguistics},
    pages={33--40},
    year={2006}
}

@inproceedings{rosti-etal:2007improved,
    title={Improved word-level system combination for machine translation},
    author={Rosti, Antti-Veikko and Matsoukas, Spyros and Schwartz, Richard},
    booktitle={Proceedings of the 45th Annual Meeting of the Association of Computational Linguistics},
    pages={312--319},
    year={2007}
}

@inproceedings{deoras-etal:2011fast,
    title={A fast re-scoring strategy to capture long-distance dependencies},
    author={Deoras, Anoop and Mikolov, Tom{\'a}{\v{s}} and Church, Kenneth},
    booktitle={Proceedings of the 2011 Conference on Empirical Methods in Natural Language Processing},
    pages={1116--1127},
    year={2011}
}

@inproceedings{khayrallah-etal:2017neural,
    title={Neural lattice search for domain adaptation in machine translation},
    author={Khayrallah, Huda and Kumar, Gaurav and Duh, Kevin and Post, Matt and Koehn, Philipp},
    booktitle={Proceedings of the Eighth International Joint Conference on Natural Language Processing (Volume 2: Short Papers)},
    pages={20--25},
    year={2017}
}

@inproceedings{stahlberg-etal:2016syntactically,
    title={Syntactically Guided Neural Machine Translation},
    author={Stahlberg, Felix and Hasler, Eva and Waite, Aurelien and Byrne, Bill},
    booktitle={Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)},
    pages={299--305},
    year={2016}
}

@inproceedings{zhou-etal:2017neural,
    title={Neural System Combination for Machine Translation},
    author={Zhou, Long and Hu, Wenpeng and Zhang, Jiajun and Zong, Chengqing},
    booktitle={Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)},
    pages={378--384},
    year={2017}
}

@inproceedings{freitag-and-al-onaizan:2017beam,
    title = "Beam Search Strategies for Neural Machine Translation",
    author = "Freitag, Markus  and Al-Onaizan, Yaser",
    booktitle = "Proceedings of the First Workshop on Neural Machine Translation",
    year = "2017",
    pages = "56--60"
}

@article{he-etal:2017decoding,
    title={Decoding with value networks for neural machine translation},
    author={He, Di and Lu, Hanqing and Xia, Yingce and Qin, Tao and Wang, Liwei and Liu, Tie-Yan},
    journal={Advances in Neural Information Processing Systems},
    volume={30},
    year={2017}
}

@article{zheng-etal:2018modeling,
    title={Modeling past and future for neural machine translation},
    author={Zheng, Zaixiang and Zhou, Hao and Huang, Shujian and Mou, Lili and Dai, Xinyu and Chen, Jiajun and Tu, Zhaopeng},
    journal={Transactions of the Association for Computational Linguistics},
    volume={6},
    pages={145--157},
    year={2018},
    publisher={MIT Press}
}

@inproceedings{ren-etal:2017deep,
    title={Deep reinforcement learning-based image captioning with embedding reward},
    author={Ren, Zhou and Wang, Xiaoyu and Zhang, Ning and Lv, Xutao and Li, Li-Jia},
    booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
    pages={290--298},
    year={2017}
}

@article{li-etal:2017learning,
    title={Learning to decode for future success},
    author={Li, Jiwei and Monroe, Will and Jurafsky, Dan},
    journal={arXiv preprint arXiv:1701.06549},
    year={2017}
}

@inproceedings{leblond-etal:2021machine,
    title = "Machine Translation Decoding beyond Beam Search",
    author="Leblond, R\'{e}mi and Alayrac, Jean-Baptiste and Sifre, Laurent and Pislar, Miruna and Jean-Baptiste, Lespiau and Antonoglou, Ioannis and Simonyan, Karen and Vinyals, Oriol",
    booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing",
    year = "2021",
    pages = "8410--8434"
}

@article{silver-etal:2017mastering,
    title={Mastering the game of go without human knowledge},
    author={Silver, David and Schrittwieser, Julian and Simonyan, Karen and Antonoglou, Ioannis and Huang, Aja and Guez, Arthur and Hubert, Thomas and Baker, Lucas and Lai, Matthew and Bolton, Adrian and Chen, Yutian and Lillicrap, Timothy and Hui, Fan and Sifre, Laurent and van den Driessche, George and Graepel, Thore and Hassabis, Demis},
    journal={nature},
    volume={550},
    number={7676},
    pages={354--359},
    year={2017},
    publisher={Nature Publishing Group}
}

@article{gulcehre-etal:2017integrating,
    title={On integrating a language model into neural machine translation},
    author={Gulcehre, Caglar and Firat, Orhan and Xu, Kelvin and Cho, Kyunghyun and Bengio, Yoshua},
    journal={Computer Speech \& Language},
    volume={45},
    pages={137--148},
    year={2017},
    publisher={Elsevier}
}

@inproceedings{edunov-etal:2018understanding,
    title={Understanding Back-Translation at Scale},
    author={Edunov, Sergey and Ott, Myle and Auli, Michael and Grangier, David},
    booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
    pages={489--500},
    year={2018}
}

@inproceedings{li-etal:2016diversity,
    title={A Diversity-Promoting Objective Function for Neural Conversation Models},
    author={Li, Jiwei and Galley, Michel and Brockett, Chris and Gao, Jianfeng and Dolan, William B},
    booktitle={Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
    pages={110--119},
    year={2016}
}

@inproceedings{ott-etal:2018analyzing,
    title={Analyzing uncertainty in neural machine translation},
    author={Ott, Myle and Auli, Michael and Grangier, David and Ranzato, Marc'Aurelio},
    booktitle={International Conference on Machine Learning},
    pages={3956--3965},
    year={2018},
    organization={PMLR}
}

@inproceedings{goodman:1996parsing,
    title={Parsing Algorithms and Metrics},
    author={Goodman, Joshua},
    booktitle={34th Annual Meeting of the Association for Computational Linguistics},
    pages={177--183},
    year={1996}
}

@article{goel-and-byrne:2000minimum,
    title={Minimum Bayes-risk automatic speech recognition},
    author={Goel, Vaibhava and Byrne, William J},
    journal={Computer Speech \& Language},
    volume={14},
    number={2},
    pages={115--135},
    year={2000},
    publisher={Elsevier}
}

@inproceedings{kumar-etal:2004minimum,
    title={Minimum Bayes-Risk Decoding for Statistical Machine Translation},
    author={Kumar, Shankar and Byrne, William},
    booktitle={Proceedings of the Human Language Technology Conference of the North American Chapter of the Association for Computational Linguistics: HLT-NAACL 2004},
    pages={169--176},
    year={2004}
}

@inproceedings{eikema-and-aziz:2020map,
    title={Is MAP Decoding All You Need? The Inadequacy of the Mode in Neural Machine Translation},
    author={Eikema, Bryan and Aziz, Wilker},
    booktitle={Proceedings of the 28th International Conference on Computational Linguistics},
    pages={4506--4520},
    year={2020}
}

@inproceedings{muller-and-sennrich:2021understanding,
    title={Understanding the Properties of Minimum Bayes Risk Decoding in Neural Machine Translation},
    author={M{\"u}ller, Mathias and Sennrich, Rico},
    booktitle={Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},
    pages={259--272},
    year={2021}
}

@article{freitag-etal:2022high,
    title={High quality rather than high model probability: Minimum Bayes risk decoding with neural metrics},
    author={Freitag, Markus and Grangier, David and Tan, Qijun and Liang, Bowen},
    journal={Transactions of the Association for Computational Linguistics},
    volume={10},
    pages={811--825},
    year={2022},
    publisher={MIT Press}
}

@inproceedings{shen-etal:2016minimum,
    title={Minimum Risk Training for Neural Machine Translation},
    author={Shen, Shiqi and Cheng, Yong and He, Zhongjun and He, Wei and Wu, Hua and Sun, Maosong and Liu, Yang},
    booktitle={Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={1683--1692},
    year={2016}
}

@article{borji-and-itti:2012state,
    title={State-of-the-art in visual attention modeling},
    author={Borji, Ali and Itti, Laurent},
    journal={IEEE transactions on pattern analysis and machine intelligence},
    volume={35},
    number={1},
    pages={185--207},
    year={2012},
    publisher={IEEE}
}

@inproceedings{xu-etal:2015show,
    title={Show, attend and tell: Neural image caption generation with visual attention},
    author={Xu, Kelvin and Ba, Jimmy and Kiros, Ryan and Cho, Kyunghyun and Courville, Aaron and Salakhudinov, Ruslan and Zemel, Rich and Bengio, Yoshua},
    booktitle={International conference on machine learning},
    pages={2048--2057},
    year={2015},
    organization={PMLR}
}

@article{jaderberg-etal:2015spatial,
    title={Spatial transformer networks},
    author = {Jaderberg, Max and Simonyan, Karen and Zisserman, Andrew and kavukcuoglu, koray},
    journal={Advances in neural information processing systems},
    volume={28},
    year={2015}
}

@article{chorowski-etal:2015attention,
    title={Attention-based models for speech recognition},
    author={Chorowski, Jan K and Bahdanau, Dzmitry and Serdyuk, Dmitriy and Cho, Kyunghyun and Bengio, Yoshua},
    journal={Advances in neural information processing systems},
    volume={28},
    year={2015}
}

@inproceedings{chan-etal:2016listen,
    title={Listen, attend and spell: A neural network for large vocabulary conversational speech recognition},
    author={Chan, William and Jaitly, Navdeep and Le, Quoc and Vinyals, Oriol},
    booktitle={2016 IEEE international conference on acoustics, speech and signal processing (ICASSP)},
    pages={4960--4964},
    year={2016},
    organization={IEEE}
}

@inproceedings{bahdanau-etal:2016end,
    title={End-to-end attention-based large vocabulary speech recognition},
    author={Bahdanau, Dzmitry and Chorowski, Jan and Serdyuk, Dmitriy and Brakel, Philemon and Bengio, Yoshua},
    booktitle={2016 IEEE international conference on acoustics, speech and signal processing (ICASSP)},
    pages={4945--4949},
    year={2016},
    organization={IEEE}
}

@misc{see:2018deep,
    Author = {Abigail See},
    url = {http://www.abigailsee.com/2018/02/21/deep-learning-structure-and-innate-priors.html},
    Month = {02},
    Title = {Deep Learning, Structure and Innate Priors: A Discussion between Yann LeCun and Christopher Manning},
    Year = {2018}
}

@inproceedings{birch-etal:2018findings,
    title = "Findings of the Second Workshop on Neural Machine Translation and Generation",
    author = "Birch, Alexandra  and
      Finch, Andrew  and
      Luong, Minh-Thang  and
      Neubig, Graham  and
      Oda, Yusuke",
    booktitle = "Proceedings of the 2nd Workshop on Neural Machine Translation and Generation",
    year = "2018",
    pages = "1--10"
}

@inproceedings{leviathan-etal:2023fast,
    title={Fast inference from transformers via speculative decoding},
    author={Leviathan, Yaniv and Kalman, Matan and Matias, Yossi},
    booktitle={Proceedings of International Conference on Machine Learning},
    pages={19274--19286},
    year={2023},
    organization={PMLR}
}

@inproceedings{heafield-etal:2021findings,
    title = "Findings of the {WMT} 2021 Shared Task on Efficient Translation",
    author = "Heafield, Kenneth  and
      Zhu, Qianqian  and
      Grundkiewicz, Roman",
    booktitle = "Proceedings of the Sixth Conference on Machine Translation",
    year = "2021",
    pages = "639--651"
}

@Misc{dao-etal:2023flashdecoding,
    howpublished = {\url{https://pytorch.org/blog/flash-decoding/}},
    note = {Retrieved 2023-10-23},
    title = {Flash-Decoding for long-context inference},
    author = {Tri Dao and Daniel Haziza and Francisco Massa and Grigory Sizov},
    year = {2023}
}

@inproceedings{anderson-etal:2017guided,
  title={Guided Open Vocabulary Image Captioning with Constrained Beam Search},
  author={Anderson, Peter and Fernando, Basura and Johnson, Mark and Gould, Stephen},
  booktitle={Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing},
  pages={936--945},
  year={2017}
}

@inproceedings{hokamp-and-liu:2017lexically,
    title={Lexically Constrained Decoding for Sequence Generation Using Grid Beam Search},
    author={Hokamp, Chris and Liu, Qun},
    booktitle={Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={1535--1546},
    year={2017}
}

@inproceedings{hasler-etal:2018neural,
    title={Neural Machine Translation Decoding with Terminology Constraints},
    author={Hasler, Eva and de Gispert, Adri{\`a} and Iglesias, Gonzalo and Byrne, Bill},
    booktitle={Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 2 (Short Papers)},
    pages={506--512},
    year={2018}
}

@inproceedings{post-etal:2018fast,
    title={Fast Lexically Constrained Decoding with Dynamic Beam Allocation for Neural Machine Translation},
    author={Post, Matt and Vilar, David},
    booktitle={Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)},
    pages={1314--1324},
    year={2018}
}

@inproceedings{kulikov-etal:2019importance,
    title={Importance of Search and Evaluation Strategies in Neural Dialogue Modeling},
    author={Kulikov, Ilia and Miller, Alexander and Cho, Kyunghyun and Weston, Jason},
    booktitle={Proceedings of the 12th International Conference on Natural Language Generation},
    pages={76--87},
    year={2019}
}

@inproceedings{liu-etal:2016agreement,
    title={Agreement on target-bidirectional neural machine translation},
    author={Liu, Lemao and Utiyama, Masao and Finch, Andrew and Sumita, Eiichiro},
    booktitle={Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
    pages={411--416},
    year={2016}
}

@inproceedings{zhang-etal:2018asynchronous,
    title={Asynchronous bidirectional decoding for neural machine translation},
    author={Zhang, Xiangwen and Su, Jinsong and Qin, Yue and Liu, Yang and Ji, Rongrong and Wang, Hongji},
    booktitle={Proceedings of the AAAI conference on artificial intelligence},
    volume={32},
    year={2018}
}

@article{zhang-etal:2020synchronous,
    title={Synchronous bidirectional inference for neural sequence generation},
    author={Zhang, Jiajun and Zhou, Long and Zhao, Yang and Zong, Chengqing},
    journal={Artificial Intelligence},
    volume={281},
    pages={103234},
    year={2020},
    publisher={Elsevier}
}

@inproceedings{ghazvininejad-etal:2019mask,
    title={Mask-Predict: Parallel Decoding of Conditional Masked Language Models},
    author={Ghazvininejad, Marjan and Levy, Omer and Liu, Yinhan and Zettlemoyer, Luke},
    booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
    pages={6112--6121},
    year={2019}
}

@inproceedings{lee-etal:2020deterministic,
    title={Deterministic non-autoregressive neural sequence modeling by iterative refinement},
    author={Lee, Jason and Mansimov, Elman and Cho, Kyunghyun},
    booktitle={2018 Conference on Empirical Methods in Natural Language Processing, EMNLP 2018},
    pages={1173--1182},
    year={2020}
}

@inproceedings{buckman-etal:2016transition,
    title={Transition-based dependency parsing with heuristic backtracking},
    author={Buckman, Jacob and Ballesteros, Miguel and Dyer, Chris},
    booktitle={Proceedings of the 2016 Conference on empirical methods in natural language processing},
    pages={2313--2318},
    year={2016}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CHAPTER 7

@inproceedings{choe-charniak:2016parsing,
    title={Parsing as language modeling},
    author={Do Kook Choe and Eugene Charniak},
    booktitle={Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing},
    pages={2331--2336},
    year={2016}
}

@inproceedings{li-etal:2017modeling,
    title={Modeling Source Syntax for Neural Machine Translation},
    author={Li, Junhui and Xiong, Deyi and Tu, Zhaopeng and Zhu, Muhua and Zhang, Min and Zhou, Guodong},
    booktitle={Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={688--697},
    year={2017}
}

@inproceedings{currey-heafield:2018multi,
    title={Multi-Source Syntactic Neural Machine Translation},
    author={Currey, Anna and Heafield, Kenneth},
    booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
    pages={2961--2966},
    year={2018}
}

@inproceedings{bai-etal:2021syntax,
    title={Syntax-BERT: Improving Pre-trained Transformers with Syntax Trees},
    author={Bai, Jiangang and Wang, Yujing and Chen, Yiren and Yang, Yaming and Bai, Jing and Yu, Jing and Tong, Yunhai},
    booktitle={Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Main Volume},
    pages={3011--3020},
    year={2021}
}

@inproceedings{strubell-etal:2018linguistically,
    title={Linguistically-Informed Self-Attention for Semantic Role Labeling},
    author={Strubell, Emma and Verga, Patrick and Andor, Daniel and Weiss, David and McCallum, Andrew},
    booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
    pages={5027--5038},
    year={2018}
}

@inproceedings{xie-etal:2017aggregated,
    title={Aggregated residual transformations for deep neural networks},
    author={Xie, Saining and Girshick, Ross and Doll{\'a}r, Piotr and Tu, Zhuowen and He, Kaiming},
    booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
    pages={1492--1500},
    year={2017}
}

@article{fan-etal:2020multi,
    title={Multi-branch attentive transformer},
    author={Fan, Yang and Xie, Shufang and Xia, Yingce and Wu, Lijun and Qin, Tao and Li, Xiang-Yang and Liu, Tie-Yan},
    journal={arXiv preprint arXiv:2006.10270},
    year={2020}
}

@inproceedings{lin-etal:2022multi,
    title={Multi-Path Transformer is Better: A Case Study on Neural Machine Translation},
    author={Lin, Ye and Zhou, Shuhan and Li, Yanyang and Ma, Anxiang and Xiao, Tong and Zhu, Jingbo},
    booktitle={Findings of the Association for Computational Linguistics: EMNLP 2022},
    pages={5646--5656},
    year={2022}
}

@inproceedings{wu-etal:2020lite,
    title={Lite Transformer with Long-Short Range Attention},
    author={Wu, Zhanghao and Liu, Zhijian and Lin, Ji and Lin, Yujun and Han, Song},
    booktitle = {Proceedings of International Conference on Learning Representations (ICLR)},
    year={2020}
}

@inproceedings{voita-etal:2018context,
    title={Context-Aware Neural Machine Translation Learns Anaphora Resolution},
    author={Voita, Elena and Serdyukov, Pavel and Sennrich, Rico and Titov, Ivan},
    booktitle={Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={1264--1274},
    year={2018}
}

@inproceedings{li-etal:2020does,
    title={Does Multi-Encoder Help? A Case Study on Context-Aware Neural Machine Translation},
    author={Li, Bei and Liu, Hui and Wang, Ziyang and Jiang, Yufan and Xiao, Tong and Zhu, Jingbo and Liu, Tongran and Li, Changliang},
    booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
    pages={3512--3518},
    year={2020}
}

@inproceedings{hao-etal:2019multi,
    title={Multi-Granularity Self-Attention for Neural Machine Translation},
    author={Hao, Jie and Wang, Xing and Shi, Shuming and Zhang, Jinfeng and Tu, Zhaopeng},
    booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
    pages={887--897},
    year={2019}
}

@inproceedings{guo-etal:2020multi,
    title={Multi-scale self-attention for text classification},
    author={Guo, Qipeng and Qiu, Xipeng and Liu, Pengfei and Xue, Xiangyang and Zhang, Zheng},
    booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
    volume={34},
    pages={7847--7854},
    year={2020}
}

@inproceedings{li-etal:2022learningmultiscale,
    title={Learning multiscale transformer models for sequence generation},
    author={Li, Bei and Zheng, Tong and Jing, Yi and Jiao, Chengbo and Xiao, Tong and Zhu, Jingbo},
    booktitle={International Conference on Machine Learning},
    pages={13225--13241},
    year={2022},
    organization={PMLR}
}

@inproceedings{fan-etal:2021multiscale,
    title={Multiscale vision transformers},
    author={Fan, Haoqi and Xiong, Bo and Mangalam, Karttikeya and Li, Yanghao and Yan, Zhicheng and Malik, Jitendra and Feichtenhofer, Christoph},
    booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
    pages={6824--6835},
    year={2021}
}

@inproceedings{li-etal:2022mvitv2,
    title={Mvitv2: Improved multiscale vision transformers for classification and detection},
    author={Li, Yanghao and Wu, Chao-Yuan and Fan, Haoqi and Mangalam, Karttikeya and Xiong, Bo and Malik, Jitendra and Feichtenhofer, Christoph},
    booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
    pages={4804--4814},
    year={2022}
}

@article{brown-etal:2020language,
    title={Language models are few-shot learners},
    author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},
    journal={Advances in neural information processing systems},
    volume={33},
    pages={1877--1901},
    year={2020}
}

@inproceedings{clark-etal:2019does,
    title={What Does BERT Look at? An Analysis of BERT's Attention},
    author={Clark, Kevin and Khandelwal, Urvashi and Levy, Omer and Manning, Christopher D},
    booktitle={Proceedings of the 2019 ACL Workshop BlackboxNLP: Analyzing and Interpreting Neural Networks for NLP},
    pages={276--286},
    year={2019}
}

@article{belinkov:2022probing,
    title = "Probing Classifiers: Promises, Shortcomings, and Advances",
    author = "Belinkov, Yonatan",
    journal = "Computational Linguistics",
    volume = "48",
    number = "1",
    year = "2022",
    address = "Cambridge, MA",
    publisher = "MIT Press",
    pages = "207--219"
}

@article{manning-etal:2020emergent,
    title={Emergent linguistic structure in artificial neural networks trained by self-supervision},
    author={Manning, Christopher D and Clark, Kevin and Hewitt, John and Khandelwal, Urvashi and Levy, Omer},
    journal={Proceedings of the National Academy of Sciences},
    volume={117},
    number={48},
    pages={30046--30054},
    year={2020},
    publisher={National Acad Sciences}
}

@inproceedings{shi-etal:2016does,
    title={Does string-based neural MT learn source syntax?},
    author={Shi, Xing and Padhi, Inkit and Knight, Kevin},
    booktitle={Proceedings of the 2016 conference on empirical methods in natural language processing},
    pages={1526--1534},
    year={2016}
}

@inproceedings{tenney-etal:2019what,
    title={What do you learn from context? Probing for sentence structure in contextualized word representations},
    author={Ian Tenney and Patrick Xia and Berlin Chen and Alex Wang and Adam Poliak and R Thomas McCoy and Najoung Kim and Benjamin Van Durme and Sam Bowman and Dipanjan Das and Ellie Pavlick},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2019}
}

@inproceedings{tenney-etal:2019bert,
    title={BERT Rediscovers the Classical NLP Pipeline},
    author={Tenney, Ian and Das, Dipanjan and Pavlick, Ellie},
    booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
    pages={4593--4601},
    year={2019}
}

@inproceedings{adifine-etal:2016fine,
    title={Fine-grained Analysis of Sentence Embeddings Using Auxiliary Prediction Tasks},
    author={Adi, Yossi and Kermany, Einat and Belinkov, Yonatan and Lavi, Ofer and Goldberg, Yoav},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2016}
}

@inproceedings{wallace-etal:2019nlp,
    title={Do NLP Models Know Numbers? Probing Numeracy in Embeddings},
    author={Wallace, Eric and Wang, Yizhong and Li, Sujian and Singh, Sameer and Gardner, Matt},
    booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
    pages={5307--5315},
    year={2019}
}

@inproceedings{kim-etal:2019probing,
    title={Probing What Different NLP Tasks Teach Machines about Function Word Comprehension},
    author={Najoung Kim and Roma Patel and Adam Poliak and Alex Wang and Patrick Xia and R. Thomas McCoy and Ian Tenney and Alexis Ross and Tal Linzen and Benjamin Van Durme and Samuel R. Bowman and Ellie Pavlick},
    booktitle={Proceedings of the Eighth Joint Conference on Lexical and Computational Semantics (* SEM 2019)},
    pages={235--249},
    year={2019}
}

@inproceedings{petroni-etal:2019language,
    title={Language Models as Knowledge Bases?},
    author={Petroni, Fabio and Rockt{\"a}schel, Tim and Riedel, Sebastian and Lewis, Patrick and Bakhtin, Anton and Wu, Yuxiang and Miller, Alexander},
    booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
    pages={2463--2473},
    year={2019}
}

@inproceedings{hewitt-etal:2019designing,
    title={Designing and Interpreting Probes with Control Tasks},
    author={Hewitt, John and Liang, Percy},
    booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
    pages={2733--2743},
    year={2019}
}

@inproceedings{sukhbaatar-etal:2019adaptive,
    title={Adaptive Attention Span in Transformers},
    author={Sukhbaatar, Sainbayar and Grave, {\'E}douard and Bojanowski, Piotr and Joulin, Armand},
    booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
    pages={331--335},
    year={2019}
}

@inproceedings{sperber-etal:2018self,
    title={Self-Attentional Acoustic Models},
    author={Sperber, Matthias and Niehues, Jan and Neubig, Graham and St{\"u}ker, Sebastian and Waibel, Alex},
    booktitle={Proceedings of Interspeech 2018},
    pages={3723--3727},
    year={2018}
}

@article{gulati-etal:2020conformer,
    title={Conformer: Convolution-augmented Transformer for Speech Recognition},
    author={Gulati, Anmol and Qin, James and Chiu, Chung-Cheng and Parmar, Niki and Zhang, Yu and Yu, Jiahui and Han, Wei and Wang, Shibo and Zhang, Zhengdong and Wu, Yonghui and Pang, Ruoming},
    journal={Proceedings of Interspeech 2020},
    pages={5036--5040},
    year={2020}
}

@inproceedings{wu-etal:2019pay,
    title={Pay Less Attention with Lightweight and Dynamic Convolutions},
    author={Wu, Felix and Fan, Angela and Baevski, Alexei and Dauphin, Yann and Auli, Michael},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2019}
}

@inproceedings{huang-etal:2018music,
    title={Music Transformer: Generating Music with Long-Term Structure},
    author={Huang, Cheng-Zhi Anna and Vaswani, Ashish and Uszkoreit, Jakob and Simon, Ian and Hawthorne, Curtis and Shazeer, Noam and Dai, Andrew M and Hoffman, Matthew D and Dinculescu, Monica and Eck, Douglas},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2018}
}

@inproceedings{dai-etal:2019transformer,
    title={Transformer-XL: Attentive Language Models beyond a Fixed-Length Context},
    author={Dai, Zihang and Yang, Zhilin and Yang, Yiming and Carbonell, Jaime G and Le, Quoc and Salakhutdinov, Ruslan},
    booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
    pages={2978--2988},
    year={2019}
}

@inproceedings{he-etal:2021deberta,
    title={DEBERTA: DECODING-ENHANCED BERT WITH DISENTANGLED ATTENTION},
    author={He, Pengcheng and Liu, Xiaodong and Gao, Jianfeng and Chen, Weizhu},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2021}
}

@inproceedings{huang-etal:2020improve,
    title={Improve Transformer Models with Better Relative Position Embeddings},
    author={Huang, Zhiheng and Liang, Davis and Xu, Peng and Xiang, Bing},
    booktitle={Findings of the Association for Computational Linguistics: EMNLP 2020},
    pages={3327--3335},
    year={2020}
}

@inproceedings{xiong-etal:2020layer,
    title={On layer normalization in the transformer architecture},
    author={Xiong, Ruibin and Yang, Yunchang and He, Di and Zheng, Kai and Zheng, Shuxin and Xing, Chen and Zhang, Huishuai and Lan, Yanyan and Wang, Liwei and Liu, Tieyan},
    booktitle={International Conference on Machine Learning},
    pages={10524--10533},
    year={2020}
}

@article{wang-etal:2022deepnet,
    title={Deepnet: Scaling transformers to 1,000 layers},
    author={Wang, Hongyu and Ma, Shuming and Dong, Li and Huang, Shaohan and Zhang, Dongdong and Wei, Furu},
    journal={arXiv preprint arXiv:2203.00555},
    year={2022}
}

@inproceedings{he-etal:2016identity,
    title={Identity mappings in deep residual networks},
    author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
    booktitle={Proceedings of ECCV 2016},
    pages={630--645},
    year={2016},
}

@article{srivastava-etal:2015highway,
    title={Highway networks},
    author={Srivastava, Rupesh Kumar and Greff, Klaus and Schmidhuber, J{\"u}rgen},
    journal={arXiv preprint arXiv:1505.00387},
    year={2015}
}

@inproceedings{liu-eta:2020understanding,
    title={Understanding the Difficulty of Training Transformers},
    author={Liu, Liyuan and Liu, Xiaodong and Gao, Jianfeng and Chen, Weizhu and Han, Jiawei},
    booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
    pages={5747--5763},
    year={2020}
}

@inproceedings{liu-etal:2020rethinking,
    title={Rethinking skip connection with layer normalization},
    author={Liu, Fenglin and Ren, Xuancheng and Zhang, Zhiyuan and Sun, Xu and Zou, Yuexian},
    booktitle={Proceedings of the 28th international conference on computational linguistics},
    pages={3586--3598},
    year={2020}
}

@article{ding-etal:2021cogview,
    title={Cogview: Mastering text-to-image generation via transformers},
    author       = {Ming Ding and Zhuoyi Yang and Wenyi Hong and Wendi Zheng and Chang Zhou and Da Yin and Junyang Lin and Xu Zou and Zhou Shao and Hongxia Yang and Jie Tang},
    journal={Advances in Neural Information Processing Systems},
    volume={34},
    pages={19822--19835},
    year={2021}
}

@article{wang-etal:2022foundation,
    title={Foundation transformers},
    author={Hongyu Wang and Shuming Ma and Shaohan Huang and Li Dong and Wenhui Wang and Zhiliang Peng and Yu Wu and Payal Bajaj and Saksham Singhal and Alon Benhaim and Barun Patra and Zhun Liu and Vishrav Chaudhary and Xia Song and Furu Wei},
    journal={arXiv preprint arXiv:2210.06423},
    year={2022}
}

@inproceedings{bachlechner-etal:2021rezero,
    title={Rezero is all you need: Fast convergence at large depth},
    author={Bachlechner, Thomas and Majumder, Bodhisattwa Prasad and Mao, Henry and Cottrell, Gary and McAuley, Julian},
    booktitle={Proceedings of Uncertainty in Artificial Intelligence},
    pages={1352--1361},
    year={2021},
    organization={PMLR}
}

@inproceedings{szegedy-etal:2014intriguing,
    title={Intriguing properties of neural networks},
    author={Szegedy, Christian and Zaremba, Wojciech and Sutskever, Ilya and Bruna, Joan and Erhan, Dumitru and Goodfellow, Ian and Fergus, Rob},
    booktitle={Proceedings of 2nd International Conference on Learning Representations (ICLR 2014)},
    year={2014}
}

@inproceedings{skorski-etal:2021revisiting,
    title={Revisiting weight initialization of deep neural networks},
    author={Skorski, Maciej and Temperoni, Alessandro and Theobald, Martin},
    booktitle={Asian Conference on Machine Learning},
    pages={1192--1207},
    year={2021},
    organization={PMLR}
}

@inproceedings{huang-etal:2017densely,
    title={Densely connected convolutional networks},
    author={Huang, Gao and Liu, Zhuang and Van Der Maaten, Laurens and Weinberger, Kilian Q},
    booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
    pages={4700--4708},
    year={2017}
}

@inproceedings{wang-etal:2018multi,
    title={Multi-layer Representation Fusion for Neural Machine Translation},
    author={Wang, Qiang and Li, Fuxue and Xiao, Tong and Li, Yanyang and Li, Yinqiao and Zhu, Jingbo},
    booktitle={Proceedings of the 27th International Conference on Computational Linguistics},
    pages={3015--3026},
    year={2018}
}

@inproceedings{huang-etal:2016deep,
    title={Deep networks with stochastic depth},
    author={Huang, Gao and Sun, Yu and Liu, Zhuang and Sedra, Daniel and Weinberger, Kilian Q},
    booktitle={Proceedings of the 14th European Conference},
    pages={646--661},
    year={2016},
    organization={Springer}
}

@inproceedings{li-etal:2021learning,
    title={Learning light-weight translation models from deep transformer},
    author={Li, Bei and Wang, Ziyang and Liu, Hui and Du, Quan and Xiao, Tong and Zhang, Chunliang and Zhu, Jingbo},
    booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
    volume={35},
    pages={13217--13225},
    year={2021}
}

@article{chen-etal:2015net2net,
    title={Net2net: Accelerating learning via knowledge transfer},
    author={Chen, Tianqi and Goodfellow, Ian and Shlens, Jonathon},
    journal={arXiv preprint arXiv:1511.05641},
    year={2015}
}

@article{ee-etal:proposal2017,
    author = {Ee, Weinan},
    year = {2017},
    month = {02},
    pages = {1-11},
    title = {A Proposal on Machine Learning via Dynamical Systems},
    volume = {5},
    journal = {Communications in Mathematics and Statistics},
}

@article{haber-ruthotto:2017stable,
    title={Stable architectures for deep neural networks},
    author={Haber, Eldad and Ruthotto, Lars},
    journal={Inverse problems},
    volume={34},
    number={1},
    pages={014004},
    year={2017},
    publisher={IOP Publishing}
}

@inproceedings{li-etal:2022ode,
    title={ODE Transformer: An Ordinary Differential Equation-Inspired Model for Sequence Generation},
    author={Li, Bei and Du, Quan and Zhou, Tao and Jing, Yi and Zhou, Shuhan and Zeng, Xin and Xiao, Tong and Zhu, Jingbo and Liu, Xuebo and Zhang, Min},
    booktitle={Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={8335--8351},
    year={2022}
}

@article{chen-etal:2018neural,
    title={Neural ordinary differential equations},
    author={Chen, Ricky TQ and Rubanova, Yulia and Bettencourt, Jesse and Duvenaud, David K},
    journal={Advances in neural information processing systems},
    volume={31},
    year={2018}
}

@article{kidger:2022neural,
    title={On neural differential equations},
    author={Kidger, Patrick},
    journal={arXiv preprint arXiv:2202.02435},
    year={2022}
}

@article{fedus-etal:2022switch,
    title={Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity},
    author={Fedus, William and Zoph, Barret and Shazeer, Noam},
    journal={The Journal of Machine Learning Research},
    volume={23},
    number={1},
    pages={5232--5270},
    year={2022},
    publisher={JMLRORG}
}

@inproceedings{lepikhing-etal:2021shard,
    title={GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding},
    author={Lepikhin, Dmitry and Lee, HyoukJoong and Xu, Yuanzhong and Chen, Dehao and Firat, Orhan and Huang, Yanping and Krikun, Maxim and Shazeer, Noam and Chen, Zhifeng},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2021}
}

@article{kaplan-etal:2020scaling,
    title={Scaling laws for neural language models},
    author={Kaplan, Jared and McCandlish, Sam and Henighan, Tom and Brown, Tom B and Chess, Benjamin and Child, Rewon and Gray, Scott and Radford, Alec and Wu, Jeffrey and Amodei, Dario},
    journal={arXiv preprint arXiv:2001.08361},
    year={2020}
}

@inproceedings{wang-etal:2023learning,
    title={Learning to Grow Pretrained Models for Efficient Transformer Training},
    author={Wang, Peihao and Panda, Rameswar and Hennigen, Lucas Torroba and Greengard, Philip and Karlinsky, Leonid and Feris, Rogerio and Cox, David Daniel and Wang, Zhangyang and Kim, Yoon},
    booktitle={Proceedings of The Eleventh International Conference on Learning Representations},
    year={2023}
}

@inproceedings{shazeer-etal:2017outrageously,
    title={Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer},
    author={Shazeer, Noam and Mirhoseini, Azalia and Maziarz, Krzysztof and Davis, Andy and Le, Quoc and Hinton, Geoffrey and Dean, Jeff},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2017}
}

@article{yuksel-etal:2012twenty,
    title={Twenty years of mixture of experts},
    author={Yuksel, Seniha Esen and Wilson, Joseph N and Gader, Paul D},
    journal={IEEE transactions on neural networks and learning systems},
    volume={23},
    number={8},
    pages={1177--1193},
    year={2012},
    publisher={IEEE}
}

@article{masoudnia-ebrahimpour:2014mixture,
    title={Mixture of experts: a literature survey},
    author={Masoudnia, Saeed and Ebrahimpour, Reza},
    journal={The Artificial Intelligence Review},
    volume={42},
    number={2},
    pages={275},
    year={2014},
    publisher={Springer Nature BV}
}

@article{chowdhery-etal:2022palm,
    title={PaLM: Scaling language modeling with pathways},
    author       = {Aakanksha Chowdhery and Sharan Narang and Jacob Devlin and Maarten Bosma and Gaurav Mishra and Adam Roberts and Paul Barham and Hyung Won Chung and Charles Sutton and Sebastian Gehrmann and Parker Schuh and Kensen Shi and Sasha Tsvyashchenko and Joshua Maynez and Abhishek Rao and Parker Barnes and Yi Tay and Noam Shazeer and Vinodkumar Prabhakaran and Emily Reif and Nan Du and Ben Hutchinson and Reiner Pope and James Bradbury and Jacob Austin and Michael Isard and Guy Gur-Ari and Pengcheng Yin and Toju Duke and Anselm Levskaya and Sanjay Ghemawat and Sunipa Dev and Henryk Michalewski and Xavier Garcia and Vedant Misra and Kevin Robinson and Liam Fedus and Denny Zhou and Daphne Ippolito and David Luan and Hyeontaek Lim and Barret Zoph and Alexander Spiridonov and Ryan Sepassi and David Dohan and Shivani Agrawal and Mark Omernick and Andrew M. Dai and Thanumalayan Sankaranarayana Pillai and Marie Pellat and Aitor Lewkowycz and Erica Moreira and Rewon Child and Oleksandr Polozov and Katherine Lee and Zongwei Zhou and Xuezhi Wang and Brennan Saeta and Mark Diaz and Orhan Firat and Michele Catasta and Jason Wei and Kathy Meier-Hellstern and Douglas Eck and Jeff Dean and Slav Petrov and Noah Fiedel},
    journal={arXiv preprint arXiv:2204.02311},
    year={2022}
}

@inproceedings{parmar-etal:2018image,
    title={Image Transformer},
    author={Parmar, Niki and Vaswani, Ashish and Uszkoreit, Jakob and Kaiser, Lukasz and Shazeer, Noam and Ku, Alexander and Tran, Dustin},
    booktitle={International conference on machine learning},
    pages={4055--4064},
    year={2018},
    organization={PMLR}
}

@inproceedings{qiu-etal:2020blockwise,
    title={Blockwise Self-Attention for Long Document Understanding},
    author={Qiu, Jiezhong and Ma, Hao and Levy, Omer and Yih, Wen-tau and Wang, Sinong and Tang, Jie},
    booktitle={Findings of the Association for Computational Linguistics: EMNLP 2020},
    pages={2555--2565},
    year={2020}
}

@article{Beltagy-etal:2020Longformer,
    title={Longformer: The Long-Document Transformer},
    author={Iz Beltagy and Matthew E. Peters and Arman Cohan},
    journal={arXiv:2004.05150},
    year={2020},
}

@inproceedings{ainslie-etal:2020etc,
    title={ETC: Encoding Long and Structured Inputs in Transformers},
    author={Ainslie, Joshua and Ontanon, Santiago and Alberti, Chris and Cvicek, Vaclav and Fisher, Zachary and Pham, Philip and Ravula, Anirudh and Sanghai, Sumit and Wang, Qifan and Yang, Li},
    booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
    pages={268--284},
    year={2020}
}

@article{child-etal:2019generating,
    title={Generating long sequences with sparse transformers},
    author={Child, Rewon and Gray, Scott and Radford, Alec and Sutskever, Ilya},
    journal={arXiv preprint arXiv:1904.10509},
    year={2019}
}

@inproceedings{zhou-etal:2021informer,
    title={Informer: Beyond efficient transformer for long sequence time-series forecasting},
    author={Zhou, Haoyi and Zhang, Shanghang and Peng, Jieqi and Zhang, Shuai and Li, Jianxin and Xiong, Hui and Zhang, Wancai},
    booktitle={Proceedings of the AAAI conference on artificial intelligence},
    volume={35},
    pages={11106--11115},
    year={2021}
}

@article{roy-etal:2021efficient,
    title={Efficient Content-Based Sparse Attention with Routing Transformers},
    author={Roy, Aurko and Saffar, Mohammad and Vaswani, Ashish and Grangier, David},
    journal={Transactions of the Association for Computational Linguistics},
    volume={9},
    pages={53--68},
    year={2021}
}

@inproceedings{kitaev-etal:2020reformer,
    title={Reformer: The Efficient Transformer},
    author={Kitaev, Nikita and Kaiser, Lukasz and Levskaya, Anselm},
    booktitle = {Proceedings of International Conference on Learning Representations},
    year={2020}
}

@inproceedings{tay-etal:2020sparse,
    title={Sparse sinkhorn attention},
    author={Tay, Yi and Bahri, Dara and Yang, Liu and Metzler, Donald and Juan, Da-Cheng},
    booktitle={Proceedings of International Conference on Machine Learning},
    pages={9438--9447},
    year={2020},
    organization={PMLR}
}

@article{zaheer-etal:2020big,
    title={Big bird: Transformers for longer sequences},
    author={Manzil Zaheer and Guru Guruganesh and Kumar Avinava Dubey and Joshua Ainslie and C. Alberti and S. Onta{\~n}{\'o}n and Philip Pham and Anirudh Ravula and Qifan Wang and L. Yang and A. Ahmed},
    journal={Advances in neural information processing systems},
    volume={33},
    pages={17283--17297},
    year={2020}
}

@inproceedings{zhang-etal:2018accelerating,
    title={Accelerating Neural Transformer via an Average Attention Network},
    author={Zhang, Biao and Xiong, Deyi and Su, Jinsong},
    booktitle={Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={1789--1798},
    year={2018}
}

@inproceedings{rae-etal:2019compressive,
    title={Compressive Transformers for Long-Range Sequence Modelling},
    author={Rae, Jack W and Potapenko, Anna and Jayakumar, Siddhant M and Hillier, Chloe and Lillicrap, Timothy P},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2019}
}

@inproceedings{martins-etal:2022former,
    title={\ensuremath{\infty}-former: Infinite Memory Transformer-former: Infinite Memory Transformer},
    author={Martins, Pedro Henrique and Marinho, Zita and Martins, Andr{\'e} FT},
    booktitle={Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={5468--5485},
    year={2022}
}

@inproceedings{wu-etal:2021memorizing,
    title={Memorizing Transformers},
    author={Wu, Yuhuai and Rabe, Markus Norman and Hutchins, DeLesley and Szegedy, Christian},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2021}
}

@inproceedings{guu-etal:2020retrieval,
    title={Retrieval augmented language model pre-training},
    author={Guu, Kelvin and Lee, Kenton and Tung, Zora and Pasupat, Panupong and Chang, Mingwei},
    booktitle={Proceedings of International conference on machine learning},
    pages={3929--3938},
    year={2020},
    organization={PMLR}
}

@article{lewis-etal:2020retrieval,
    title={Retrieval-augmented generation for knowledge-intensive nlp tasks},
    author={Patrick Lewis and Ethan Perez and Aleksandra Piktus and Fabio Petroni and Vladimir Karpukhin and Naman Goyal and Heinrich K{\"{u}}ttler and Mike Lewis and Wen-tau Yih and Tim Rockt{\"{a}}schel and Sebastian Riedel and Douwe Kiela},
    journal={Advances in Neural Information Processing Systems},
    volume={33},
    pages={9459--9474},
    year={2020}
}

@inproceedings{liu-etal:2018generating,
    title={Generating Wikipedia by Summarizing Long Sequences},
    author={Liu, Peter J and Saleh, Mohammad and Pot, Etienne and Goodrich, Ben and Sepassi, Ryan and Kaiser, Lukasz and Shazeer, Noam},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2018}
}

@article{lample-etal:2019large,
    title={Large memory layers with product keys},
    author={Lample, Guillaume and Sablayrolles, Alexandre and Ranzato, Marc'Aurelio and Denoyer, Ludovic and J{\'e}gou, Herv{\'e}},
    journal={Advances in Neural Information Processing Systems},
    volume={32},
    year={2019}
}

@inproceedings{choromanski-etal:2020rethinking,
    title={Rethinking Attention with Performers},
    author       = {Krzysztof Marcin Choromanski and Valerii Likhosherstov and David Dohan and Xingyou Song and Andreea Gane and Tam{\'{a}}s Sarl{\'{o}}s and Peter Hawkins and Jared Quincy Davis and Afroz Mohiuddin and Lukasz Kaiser and David Benjamin Belanger and Lucy J. Colwell and Adrian Weller},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2020}
}

@article{wang-etal:2020linformer,
    title={Linformer: Self-attention with linear complexity},
    author={Wang, Sinong and Li, Belinda Z and Khabsa, Madian and Fang, Han and Ma, Hao},
    journal={arXiv preprint arXiv:2006.04768},
    year={2020}
}

@inproceedings{katharopoulos-etal:2020transformers,
    title={Transformers are RNNs: Fast autoregressive transformers with linear attention},
    author={Katharopoulos, Angelos and Vyas, Apoorv and Pappas, Nikolaos and Fleuret, Fran{\c{c}}ois},
    booktitle={International conference on machine learning},
    pages={5156--5165},
    year={2020},
    organization={PMLR}
}

@article{pires-etal:2023one,
    title={One Wide Feedforward is All You Need},
    author={Pires, Telmo Pessoa and Lopes, Ant{\'o}nio V and Assogba, Yannick and Setiawan, Hendra},
    journal={arXiv preprint arXiv:2309.01826},
    year={2023}
}

@article{shazeer:2019fast,
    title={Fast transformer decoding: One write-head is all you need},
    author={Shazeer, Noam},
    journal={arXiv preprint arXiv:1911.02150},
    year={2019}
}

@inproceedings{xiao-etal:2019sharing,
    title={Sharing Attention Weights for Fast Transformer},
    author={Xiao, Tong and Li, Yinqiao and Zhu, Jingbo and Yu, Zhengtao and Liu, Tongran},
    booktitle={Proceedings of the Twenty-Eighth International Joint Conference on Artificial Intelligence (IJCAI-19)},
    pages={5292--5298},
    year={2019}
}

@article{gomez-etal:2017reversible,
    title={The reversible residual network: Backpropagation without storing activations},
    author={Gomez, Aidan N and Ren, Mengye and Urtasun, Raquel and Grosse, Roger B},
    journal={Advances in neural information processing systems},
    volume={30},
    year={2017}
}

@inproceedings{wu-etal:2018pay,
    title={Pay Less Attention with Lightweight and Dynamic Convolutions},
    author={Wu, Felix and Fan, Angela and Baevski, Alexei and Dauphin, Yann and Auli, Michael},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2018}
}

@inproceedings{chen-etal:2018best,
    title = "The Best of Both Worlds: Combining Recent Advances in Neural Machine Translation",
    author = "Chen, Mia Xu and Firat, Orhan and Bapna, Ankur and Johnson, Melvin and Macherey, Wolfgang and Foster, George and Jones, Llion and Schuster, Mike and Shazeer, Noam and Parmar, Niki and Vaswani, Ashish and Uszkoreit, Jakob and Kaiser, Lukasz and Chen, Zhifeng and Wu, Yonghui and Hughes, Macduff",
    booktitle = "Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    year = "2018",
    pages = "76--86"
}

@inproceedings{peng-etal:2021random,
    title={Random Feature Attention},
    author={Peng, H and Pappas, N and Yogatama, D and Schwartz, R and Smith, N and Kong, L},
    booktitle={Proceedings of International Conference on Learning Representations (ICLR 2021)},
    year={2021}
}

@inproceedings{bello:2020lambdanetworks,
    title={LambdaNetworks: Modeling long-range Interactions without Attention},
    author={Bello, Irwan},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2020}
}

@inproceedings{schlag-etal:2021linear,
    title={Linear transformers are secretly fast weight programmers},
    author={Schlag, Imanol and Irie, Kazuki and Schmidhuber, J{\"u}rgen},
    booktitle={Proceedings of International Conference on Machine Learning},
    pages={9355--9366},
    year={2021},
    organization={PMLR}
}

@article{peng-etal:2023rwkv,
    title={RWKV: Reinventing RNNs for the Transformer Era},
    author       = {Bo Peng and Eric Alcaide and Quentin Anthony and Alon Albalak and Samuel Arcadinho and Stella Biderman and Huanqi Cao and Xin Cheng and Michael Chung and Leon Derczynski and Xingjian Du and Matteo Grella and Kranthi Gv and Xuzheng He and Haowen Hou and Przemyslaw Kazienko and Jan Kocon and Jiaming Kong and Bartlomiej Koptyra and Hayden Lau and Jiaju Lin and Krishna Sri Ipsit Mantri and Ferdinand Mom and Atsushi Saito and Guangyu Song and Xiangru Tang and Johan S. Wind and Stanislaw Wozniak and Zhenyuan Zhang and Qinghua Zhou and Jian Zhu and Rui{-}Jie Zhu},
    journal={arXiv preprint arXiv:2305.13048},
    year={2023}
}

@article{sun-etal:2023retentive,
    title={Retentive Network: A Successor to Transformer for Large Language Models},
    author={Sun, Yutao and Dong, Li and Huang, Shaohan and Ma, Shuming and Xia, Yuqing and Xue, Jilong and Wang, Jianyong and Wei, Furu},
    journal={arXiv preprint arXiv:2307.08621},
    year={2023}
}

@Misc{gu-etal:2022ssms-blog,
    howpublished = {\url{https://hazyresearch.stanford.edu/blog/2022-01-14-s4-3}},
    note = {Retrieved 2022-01-14},
    title = {Structured State Spaces: Combining Continuous-Time, Recurrent, and Convolutional Models},
    author = {Albert Gu and Karan Goel and Khaled Saab and Chris R\'{e}},
    year = {2022}
}

@book{aastrom-and-wittenmark:2013computer,
    title={Computer-controlled systems: theory and design},
    author={{\AA}str{\"o}m, Karl J and Wittenmark, Bj{\"o}rn},
    year={2013},
    publisher={Courier Corporation}
}

@inproceedings{gu-etal:2021efficiently,
    title={Efficiently Modeling Long Sequences with Structured State Spaces},
    author={Gu, Albert and Goel, Karan and R{\'e}, Christopher},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2021}
}

@article{gu-etal:2022parameterization,
    title={On the parameterization and initialization of diagonal state space models},
    author={Gu, Albert and Goel, Karan and Gupta, Ankit and R{\'e}, Christopher},
    journal={Advances in Neural Information Processing Systems},
    volume={35},
    pages={35971--35983},
    year={2022}
}

@article{orvieto-etal:2023resurrecting,
    title={Resurrecting recurrent neural networks for long sequences},
    author={Orvieto, Antonio and Smith, Samuel L and Gu, Albert and Fernando, Anushan and Gulcehre, Caglar and Pascanu, Razvan and De, Soham},
    journal={arXiv preprint arXiv:2303.06349},
    year={2023}
}

@inproceedings{fu-etal:2022hungry,
    title={Hungry Hungry Hippos: Towards Language Modeling with State Space Models},
    author={Fu, Daniel Y and Dao, Tri and Saab, Khaled Kamal and Thomas, Armin W and Rudra, Atri and Re, Christopher},
    booktitle={Proceedings of The Eleventh International Conference on Learning Representations},
    year={2022}
}

@article{bengio-etal:2013estimating,
    title={Estimating or propagating gradients through stochastic neurons for conditional computation},
    author={Bengio, Yoshua and L{\'e}onard, Nicholas and Courville, Aaron},
    journal={arXiv preprint arXiv:1308.3432},
    year={2013}
}

@article{elsken-etal:2019neural,
    title={Neural architecture search: A survey},
    author={Elsken, Thomas and Metzen, Jan Hendrik and Hutter, Frank},
    journal={The Journal of Machine Learning Research},
    volume={20},
    number={1},
    pages={1997--2017},
    year={2019},
    publisher={JMLR. org}
}

@inproceedings{zoph-and-Le:2016neural,
    title={Neural Architecture Search with Reinforcement Learning},
    author={Zoph, Barret and Le, Quoc},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2016}
}

@inproceedings{howard-etal:2019searching,
    title={Searching for mobilenetv3},
    author={Andrew Howard and Ruoming Pang and Hartwig Adam and Quoc V. Le and Mark Sandler and Bo Chen and Weijun Wang and Liang-Chieh Chen and Mingxing Tan and Grace Chu and Vijay Vasudevan and Yukun Zhu},
    booktitle={Proceedings of the IEEE/CVF international conference on computer vision},
    pages={1314--1324},
    year={2019}
}

@inproceedings{tan-etal:2019efficientnet,
    title={Efficientnet: Rethinking model scaling for convolutional neural networks},
    author={Tan, Mingxing and Le, Quoc},
    booktitle={International conference on machine learning},
    pages={6105--6114},
    year={2019},
    organization={PMLR}
}

@book{gupta-etal:2004static,
    title={Static and dynamic neural networks: from fundamentals to advanced theory},
    author={Gupta, Madan and Jin, Liang and Homma, Noriyasu},
    year={2004},
    publisher={John Wiley \& Sons}
}

@article{han-etal:2021dynamic,
    title={Dynamic neural networks: A survey},
    author={Han, Yizeng and Huang, Gao and Song, Shiji and Yang, Le and Wang, Honghui and Wang, Yulin},
    journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},
    volume={44},
    number={11},
    pages={7436--7456},
    year={2021},
    publisher={IEEE}
}

@inproceedings{wang-etal:2020hat,
    title={HAT: Hardware-Aware Transformers for Efficient Natural Language Processing},
    author={Wang, Hanrui and Wu, Zhanghao and Liu, Zhijian and Cai, Han and Zhu, Ligeng and Gan, Chuang and Han, Song},
    booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
    pages={7675--7688},
    year={2020}
}

@inproceedings{so-etal:2019evolved,
    title={The evolved transformer},
    author={So, David and Le, Quoc and Liang, Chen},
    booktitle={Proceedings of International conference on machine learning},
    pages={5877--5886},
    year={2019},
    organization={PMLR}
}

@inproceedings{hu-etal:2021ranknas,
    title={RankNAS: Efficient Neural Architecture Search by Pairwise Ranking},
    author={Hu, Chi and Wang, Chenglong and Ma, Xiangnan and Meng, Xia and Li, Yinqiao and Xiao, Tong and Zhu, Jingbo and Li, Changliang},
    booktitle={Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},
    pages={2469--2480},
    year={2021}
}

@article{bengio-etal:2015conditional,
    title={Conditional computation in neural networks for faster models},
    author={Bengio, Emmanuel and Bacon, Pierre-Luc and Pineau, Joelle and Precup, Doina},
    journal={arXiv preprint arXiv:1511.06297},
    year={2015}
}

@inproceedings{xu-and-mcauley:2023survey,
    title={A Survey on Dynamic Neural Networks for Natural Language Processing},
    author={Xu, Canwen and Mcauley, Julian},
    booktitle={Findings of the Association for Computational Linguistics: EACL 2023},
    pages={2325--2336},
    year={2023}
}

@article{shen-etal:2020simple,
    title={A simple but tough-to-beat data augmentation approach for natural language understanding and generation},
    author={Shen, Dinghan and Zheng, Mingzhi and Shen, Yelong and Qu, Yanru and Chen, Weizhu},
    journal={arXiv preprint arXiv:2009.13818},
    year={2020}
}

@inproceedings{kim-and-cho:2021length,
    title = "Length-Adaptive Transformer: Train Once with Length Drop, Use Anytime with Search",
    author = "Kim, Gyuwan  and Cho, Kyunghyun",
    booktitle = "Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)",
    year = "2021",
    pages = "6501--6511"
}

@article{fedus-etal:2022review,
    title={A review of sparse expert models in deep learning},
    author={Fedus, William and Dean, Jeff and Zoph, Barret},
    journal={arXiv preprint arXiv:2209.01667},
    year={2022}
}

@inproceedings{teerapittayanon-etal:2016branchynet,
    title={Branchynet: Fast inference via early exiting from deep neural networks},
    author={Teerapittayanon, Surat and McDanel, Bradley and Kung, Hsiang-Tsung},
    booktitle={2016 23rd international conference on pattern recognition (ICPR)},
    pages={2464--2469},
    year={2016},
    organization={IEEE}
}

@inproceedings{xin-etal:2020deebert,
    title={DeeBERT: Dynamic Early Exiting for Accelerating BERT Inference},
    author={Xin, Ji and Tang, Raphael and Lee, Jaejun and Yu, Yaoliang and Lin, Jimmy},
    booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
    pages={2246--2251},
    year={2020}
}

@article{zhou-etal:2020bert,
    title={Bert loses patience: Fast and robust inference with early exit},
    author={Zhou, Wangchunshu and Xu, Canwen and Ge, Tao and McAuley, Julian and Xu, Ke and Wei, Furu},
    journal={Advances in Neural Information Processing Systems},
    volume={33},
    pages={18330--18341},
    year={2020}
}

@article{schuster-etal:2022confident,
    title={Confident adaptive language modeling},
    author={Schuster, Tal and Fisch, Adam and Gupta, Jai and Dehghani, Mostafa and Bahri, Dara and Tran, Vinh and Tay, Yi and Metzler, Donald},
    journal={Advances in Neural Information Processing Systems},
    volume={35},
    pages={17456--17472},
    year={2022}
}

@inproceedings{elbayad-etal:2020depth,
    title={Depth-Adaptive Transformer},
    author={Elbayad, Maha and Gu, Jiatao and Grave, Edouard and Auli, Michael},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2020}
}

@inproceedings{schwartz-etal:2020right,
    title={The Right Tool for the Job: Matching Model and Instance Complexities},
    author={Schwartz, Roy and Stanovsky, Gabriel and Swayamdipta, Swabha and Dodge, Jesse and Smith, Noah A},
    booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
    pages={6640--6651},
    year={2020}
}

@article{sun-etal:2021early,
    title={Early exiting with ensemble internal classifiers},
    author={Sun, Tianxiang and Zhou, Yunhua and Liu, Xiangyang and Zhang, Xinyu and Jiang, Hao and Cao, Zhao and Huang, Xuanjing and Qiu, Xipeng},
    journal={arXiv preprint arXiv:2105.13792},
    year={2021}
}

@inproceedings{liao-etal:2021global,
    title={A global past-future early exit method for accelerating inference of pre-trained language models},
    author={Liao, Kaiyuan and Zhang, Yi and Ren, Xuancheng and Su, Qi and Sun, Xu and He, Bin},
    booktitle={Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
    pages={2013--2023},
    year={2021}
}

@inproceedings{wu-etal:2018blockdrop,
    title={Blockdrop: Dynamic inference paths in residual networks},
    author={Wu, Zuxuan and Nagarajan, Tushar and Kumar, Abhishek and Rennie, Steven and Davis, Larry S and Grauman, Kristen and Feris, Rogerio},
    booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
    pages={8817--8826},
    year={2018}
}

@inproceedings{wang-etal:2018skipnet,
    title={Skipnet: Learning dynamic routing in convolutional networks},
    author={Wang, Xin and Yu, Fisher and Dou, Zi-Yi and Darrell, Trevor and Gonzalez, Joseph E},
    booktitle={Proceedings of the European Conference on Computer Vision (ECCV)},
    pages={409--424},
    year={2018}
}

@inproceedings{wang-etal:2022skipbert,
    title={Skipbert: Efficient inference with shallow layer skipping},
    author={Wang, Jue and Chen, Ke and Chen, Gang and Shou, Lidan and McAuley, Julian},
    booktitle={Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={7287--7301},
    year={2022}
}

@article{del-etal:2023skipdecode,
    title={SkipDecode: Autoregressive Skip Decoding with Batching and Caching for Efficient LLM Inference},
    author={Del Corro, Luciano and Del Giorno, Allie and Agarwal, Sahaj and Yu, Bin and Awadallah, Ahmed and Mukherjee, Subhabrata},
    journal={arXiv preprint arXiv:2307.02628},
    year={2023}
}

@article{hinton-etal:2015distilling,
    title={Distilling the knowledge in a neural network},
    author={Hinton, Geoffrey and Vinyals, Oriol and Dean, Jeff},
    journal={arXiv preprint arXiv:1503.02531},
    year={2015}
}

@article{romero-etal:2014fitnets,
    title={Fitnets: Hints for thin deep nets},
    author={Romero, Adriana and Ballas, Nicolas and Kahou, Samira Ebrahimi and Chassang, Antoine and Gatta, Carlo and Bengio, Yoshua},
    journal={arXiv preprint arXiv:1412.6550},
    year={2014}
}

@inproceedings{park-etal:2019relational,
    title={Relational knowledge distillation},
    author={Park, Wonpyo and Kim, Dongju and Lu, Yan and Cho, Minsu},
    booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
    pages={3967--3976},
    year={2019}
}

@inproceedings{peng-etal:2019correlation,
    title={Correlation congruence for knowledge distillation},
    author={Peng, Baoyun and Jin, Xiao and Liu, Jiaheng and Li, Dongsheng and Wu, Yichao and Liu, Yu and Zhou, Shunfeng and Zhang, Zhaoning},
    booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
    pages={5007--5016},
    year={2019}
}

@inproceedings{kim-etal:2016sequence,
    title={Sequence-Level Knowledge Distillation},
    author={Kim, Yoon and Rush, Alexander M},
    booktitle={Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing},
    pages={1317--1327},
    year={2016}
}

@article{gou-etal:2021knowledge,
    title={Knowledge distillation: A survey},
    author={Gou, Jianping and Yu, Baosheng and Maybank, Stephen J and Tao, Dacheng},
    journal={International Journal of Computer Vision},
    volume={129},
    pages={1789--1819},
    year={2021},
    publisher={Springer}
}

@article{wang-and-yoon:2021knowledge,
    title={Knowledge distillation and student-teacher learning for visual intelligence: A review and new outlooks},
    author={Wang, Lin and Yoon, Kuk-Jin},
    journal={IEEE transactions on pattern analysis and machine intelligence},
    volume={44},
    number={6},
    pages={3048--3068},
    year={2021},
    publisher={IEEE}
}

@inproceedings{voita-etal:2019analyzing,
    title={Analyzing Multi-Head Self-Attention: Specialized Heads Do the Heavy Lifting, the Rest Can Be Pruned},
    author={Voita, Elena and Talbot, David and Moiseev, Fedor and Sennrich, Rico and Titov, Ivan},
    booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
    pages={5797--5808},
    year={2019}
}

@article{michel-etal:2019sixteen,
    title={Are sixteen heads really better than one?},
    author={Michel, Paul and Levy, Omer and Neubig, Graham},
    journal={Advances in neural information processing systems},
    volume={32},
    year={2019}
}

@article{hou-etal:2020dynabert,
    title={Dynabert: Dynamic bert with adaptive width and depth},
    author={Hou, Lu and Huang, Zhiqi and Shang, Lifeng and Jiang, Xin and Chen, Xiao and Liu, Qun},
    journal={Advances in Neural Information Processing Systems},
    volume={33},
    pages={9782--9793},
    year={2020}
}

@inproceedings{kim-and-awadalla:2020fastformers,
    title={FastFormers: Highly Efficient Transformer Models for Natural Language Understanding},
    author={Kim, Young Jin and Awadalla, Hany Hassan},
    booktitle={Proceedings of SustaiNLP: Workshop on Simple and Efficient Natural Language Processing},
    pages={149--158},
    year={2020}
}

@article{santacroce-etal:2023matters,
    title={What Matters In The Structured Pruning of Generative Language Models?},
    author={Santacroce, Michael and Wen, Zixin and Shen, Yelong and Li, Yuanzhi},
    journal={arXiv preprint arXiv:2302.03773},
    year={2023}
}

@article{mccarley-etal:2019structured,
    title={Structured pruning of a bert-based question answering model},
    author={McCarley, JS and Chakravarti, Rishav and Sil, Avirup},
    journal={arXiv preprint arXiv:1910.06360},
    year={2019}
}

@inproceedings{wang-etal:2020structured,
    title={Structured Pruning of Large Language Models},
    author={Wang, Ziheng and Wohlwend, Jeremy and Lei, Tao},
    booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
    pages={6151--6162},
    year={2020}
}

@inproceedings{lagunas-etal:2021block,
    title={Block Pruning For Faster Transformers},
    author={Lagunas, Fran{\c{c}}ois and Charlaix, Ella and Sanh, Victor and Rush, Alexander M},
    booktitle={Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},
    pages={10619--10629},
    year={2021}
}

@inproceedings{jaegle-etal:2021perceiver,
    title={Perceiver IO: A General Architecture for Structured Inputs \& Outputs},
    author={Andrew Jaegle and Sebastian Borgeaud and Jean{-}Baptiste Alayrac and Carl Doersch and Catalin Ionescu and David Ding and Skanda Koppula and Daniel Zoran and Andrew Brock and Evan Shelhamer and Olivier J. H{\'{e}}naff and Matthew M. Botvinick and Andrew Zisserman and Oriol Vinyals and Jo{\~{a}}o Carreira},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2021}
}

@inproceedings{burchi-and-vielzeuf:2021efficient,
    title={Efficient conformer: Progressive downsampling and grouped attention for automatic speech recognition},
    author={Burchi, Maxime and Vielzeuf, Valentin},
    booktitle={Proceedings of 2021 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)},
    pages={8--15},
    year={2021},
    organization={IEEE}
}

@inproceedings{han-etal:2020contextnet,
    title={ContextNet: Improving Convolutional Neural Networks for Automatic Speech Recognition with Global Context},
    author={Han, Wei and Zhang, Zhengdong and Zhang, Yu and Yu, Jiahui and Chiu, Chung-Cheng and Qin, James and Gulati, Anmol and Pang, Ruoming and Wu, Yonghui},
    booktitle={Proceedings of Interspeech 2020},
    pages={3610--3614},
    year={2020}
}

@inproceedings{xu-etal:2023bridging,
    title = "Bridging the Granularity Gap for Acoustic Modeling",
    author = "Xu, Chen and Zhang, Yuhao and Jiao, Chengbo and Liu, Xiaoqian and Hu, Chi and Zeng, Xin and Xiao, Tong and Ma, Anxiang and Wang, Huizhen and Zhu, Jingbo",
    booktitle = "Findings of the Association for Computational Linguistics: ACL 2023",
    year = "2023",
    pages = "10816--10833"
}

@inproceedings{provilkov-etal:2020bpe,
    title={BPE-Dropout: Simple and Effective Subword Regularization},
    author={Provilkov, Ivan and Emelianenko, Dmitrii and Voita, Elena},
    booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
    pages={1882--1892},
    year={2020}
}

@article{kim-etal:2023full,
    title = {Full stack optimization of transformer inference: a survey},
    author = {Sehoon Kim and Coleman Hooper and Thanakul Wattanawong and Minwoo Kang and Ruohan Yan and Hasan Genc and Grace Dinh and Qijing Huang and Kurt Keutzer and Michael W. Mahoney and Yakun Sophia Shao and Amir Gholami},
    journal = {arXiv preprint arXiv:2302.14017},
    year = {2023}
}

@inproceedings{barham-etal:2022pathways,
    title={Pathways: Asynchronous distributed dataflow for ml},
    author       = {Paul Barham and Aakanksha Chowdhery and Jeff Dean and Sanjay Ghemawat and Steven Hand and Dan Hurt and Michael Isard and Hyeontaek Lim and Ruoming Pang and Sudip Roy and Brennan Saeta and Parker Schuh and Ryan Sepassi and Laurent El Shafey and Chandramohan A. Thekkath and Yonghui Wu},
    booktitle={Proceedings of Machine Learning and Systems},
    volume={4},
    pages={430--449},
    year={2022}
}

@article{oppenheim-and-schafer:1975digital,
    title={Digital signal processing(Book)},
    author={Oppenheim, Alan V and Schafer, Ronald W},
    journal={Prentice-Hall},
    year={1975}
}

@article{rabiner-gold:1975theory,
    title={Theory and application of digital signal processing},
    author={Rabiner, Lawrence R and Gold, Bernard},
    journal={Prentice-Hall},
    year={1975}
}

@article{gray-etal:1998quantization,
    title={Quantization},
    author={Robert M. Gray},
    journal={IEEE transactions on information theory},
    volume={44},
    number={6},
    pages={2325--2383},
    year={1998},
    publisher={IEEE}
}

@incollection{gholami-etal:2022survey,
    title={A survey of quantization methods for efficient neural network inference},
    author={Gholami, Amir and Kim, Sehoon and Dong, Zhen and Yao, Zhewei and Mahoney, Michael W and Keutzer, Kurt},
    booktitle={Low-Power Computer Vision},
    pages={291--326},
    year={2022},
    publisher={Chapman and Hall/CRC}
}

@inproceedings{jacob-etal:2018quantization,
    title={Quantization and training of neural networks for efficient integer-arithmetic-only inference},
    author={Jacob, Benoit and Kligys, Skirmantas and Chen, Bo and Zhu, Menglong and Tang, Matthew and Howard, Andrew and Adam, Hartwig and Kalenichenko, Dmitry},
    booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
    pages={2704--2713},
    year={2018}
}

@article{nagel-etal:2021white,
    title={A white paper on neural network quantization},
    author={Nagel, Markus and Fournarakis, Marios and Amjad, Rana Ali and Bondarenko, Yelysei and Van Baalen, Mart and Blankevoort, Tijmen},
    journal={arXiv preprint arXiv:2106.08295},
    year={2021}
}

@inproceedings{bondarenko-etal:2021understanding,
    title={Understanding and Overcoming the Challenges of Efficient Transformer Quantization},
    author={Bondarenko, Yelysei and Nagel, Markus and Blankevoort, Tijmen},
    booktitle={Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},
    pages={7947--7969},
    year={2021}
}

@article{yang-etal:2023quantization,
    title={Quantization-aware and tensor-compressed training of transformers for natural language understanding},
    author={Yang, Zi and Choudhary, Samridhi and Kunzmann, Siegfried and Zhang, Zheng},
    journal={arXiv preprint arXiv:2306.01076},
    year={2023}
}

@inproceedings{stock-etal:2021training,
    title={Training with Quantization Noise for Extreme Model Compression},
    author={Stock, Pierre and Fan, Angela and Graham, Benjamin and Grave, Edouard and Gribonval, R{\'e}mi and Jegou, Herve and Joulin, Armand},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2021}
}

@inproceedings{ivanov-etal:2021data,
    title={Data movement is all you need: A case study on optimizing transformers},
    author={Ivanov, Andrei and Dryden, Nikoli and Ben-Nun, Tal and Li, Shigang and Hoefler, Torsten},
    booktitle={Proceedings of Machine Learning and Systems},
    volume={3},
    pages={711--732},
    year={2021}
}

@article{dao-etal:2022flashattention,
    title={Flashattention: Fast and memory-efficient exact attention with io-awareness},
    author={Dao, Tri and Fu, Dan and Ermon, Stefano and Rudra, Atri and R{\'e}, Christopher},
    journal={Advances in Neural Information Processing Systems},
    volume={35},
    pages={16344--16359},
    year={2022}
}

@article{kwon-etal:2023efficient,
    title={Efficient memory management for large language model serving with pagedattention},
    author={Kwon, Woosuk and Li, Zhuohan and Zhuang, Siyuan and Sheng, Ying and Zheng, Lianmin and Yu, Cody Hao and Gonzalez, Joseph E and Zhang, Hao and Stoica, Ion},
    journal={arXiv preprint arXiv:2309.06180},
    year={2023}
}

@inproceedings{pope-etal:2023efficiently,
    title={Efficiently scaling transformer inference},
    author={Pope, Reiner and Douglas, Sholto and Chowdhery, Aakanksha and Devlin, Jacob and Bradbury, James and Heek, Jonathan and Xiao, Kefan and Agrawal, Shivani and Dean, Jeff},
    booktitle={Proceedings of Machine Learning and Systems},
    year={2023}
}

@inproceedings{reimers-etal:2019sentence,
    title={Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks},
    author={Reimers, Nils and Gurevych, Iryna},
    booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
    pages={3982--3992},
    year={2019}
}

@inproceedings{di-etal:2019enhancing,
    title={Enhancing transformer for end-to-end speech-to-text translation},
    author={Di Gangi, Mattia Antonino and Negri, Matteo and Cattoni, Roldano and Dessi, Roberto and Turchi, Marco},
    booktitle={Proceedings of Machine Translation Summit XVII: Research Track},
    pages={21--31},
    year={2019}
}

@article{liu-etal:2020bridging,
    title={Bridging the modality gap for speech-to-text translation},
    author={Liu, Yuchen and Zhu, Junnan and Zhang, Jiajun and Zong, Chengqing},
    journal={arXiv preprint arXiv:2010.14920},
    year={2020}
}

@inproceedings{xu-etal:2021stacked,
    title={Stacked Acoustic-and-Textual Encoding: Integrating the Pre-trained Models into Speech Translation Encoders},
    author={Xu, Chen and Hu, Bojie and Li, Yanyang and Zhang, Yuhao and Huang, Shen and Ju, Qi and Xiao, Tong and Zhu, Jingbo},
    booktitle={Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},
    pages={2619--2630},
    year={2021}
}

@article{ye-etal:2021end,
    title={End-to-end speech translation via cross-modal progressive training},
    author={Ye, Rong and Wang, Mingxuan and Li, Lei},
    journal={arXiv preprint arXiv:2104.10380},
    year={2021}
}

@inproceedings{xu-etal:2023recent,
    title={Recent Advances in Direct Speech-to-text Translation},
    author={Chen Xu and Rong Ye and Qianqian Dong and Chengqi Zhao and Tom Ko and Mingxuan Wang and Tong Xiao and Jingbo Zhu},
    booktitle={Proceedings of the Thirty-Second International Joint Conference on Artificial Intelligence (IJCAI-23): Survey Track},
    pages={6796--6804},
    year={2023}
}

@article{han-etal:2022survey,
    title={A survey on vision transformer},
    author={Kai Han and Yunhe Wang and Hanting Chen and Xinghao Chen and Jianyuan Guo and Zhenhua Liu and Yehui Tang and An Xiao and Chunjing Xu and Yixing Xu and Zhaohui Yang and Yiman Zhang and Dacheng Tao},
    journal={IEEE transactions on pattern analysis and machine intelligence},
    volume={45},
    number={1},
    pages={87--110},
    year={2022},
    publisher={IEEE}
}

@article{liu-etal:2023survey,
    title={A survey of visual transformers},
    author={Liu, Yang and Zhang, Yao and Wang, Yixin and Hou, Feng and Yuan, Jin and Tian, Jiang and Zhang, Yang and Shi, Zhongchao and Fan, Jianping and He, Zhiqiang},
    journal={IEEE Transactions on Neural Networks and Learning Systems},
    year={2023},
    publisher={IEEE}
}

@inproceedings{caron-etal:2021emerging,
    title={Emerging properties in self-supervised vision transformers},
    author={Caron, Mathilde and Touvron, Hugo and Misra, Ishan and J{\'e}gou, Herv{\'e} and Mairal, Julien and Bojanowski, Piotr and Joulin, Armand},
    booktitle={Proceedings of the IEEE/CVF international conference on computer vision},
    pages={9650--9660},
    year={2021}
}

@article{xu-etal:2023multimodal,
    title={Multimodal learning with transformers: A survey},
    author={Xu, Peng and Zhu, Xiatian and Clifton, David A},
    journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},
    year={2023},
    publisher={IEEE}
}

@inproceedings{li-etal:2022blip,
    title={Blip: Bootstrapping language-image pre-training for unified vision-language understanding and generation},
    author={Li, Junnan and Li, Dongxu and Xiong, Caiming and Hoi, Steven},
    booktitle={International Conference on Machine Learning},
    pages={12888--12900},
    year={2022},
    organization={PMLR}
}

@article{alayrac-etal:2022flamingo,
    title={Flamingo: a visual language model for few-shot learning},
    author={Jean-Baptiste Alayrac and Jeff Donahue and Pauline Luc and Antoine Miech and Iain Barr and Yana Hasson and Karel Lenc and Arthur Mensch and Katie Millican and Malcolm Reynolds and Roman Ring and Eliza Rutherford and Serkan Cabi and Tengda Han and Zhitao Gong and Sina Samangooei and Marianne Monteiro and Jacob Menick and Sebastian Borgeaud and Andrew Brock and Aida Nematzadeh and Sahand Sharifzadeh and Mikolaj Binkowski and Ricardo Barreira and Oriol Vinyals and Andrew Zisserman and Karen Simonyan},
    journal={Advances in Neural Information Processing Systems},
    volume={35},
    pages={23716--23736},
    year={2022}
}

@inproceedings{cho-etal:2021unifying,
    title={Unifying vision-and-language tasks via text generation},
    author={Cho, Jaemin and Lei, Jie and Tan, Hao and Bansal, Mohit},
    booktitle={International Conference on Machine Learning},
    pages={1931--1942},
    year={2021},
    organization={PMLR}
}

@inproceedings{antol-etal:2015vqa,
    title={Vqa: Visual question answering},
    author={Antol, Stanislaw and Agrawal, Aishwarya and Lu, Jiasen and Mitchell, Margaret and Batra, Dhruv and Zitnick, C Lawrence and Parikh, Devi},
    booktitle={Proceedings of the IEEE international conference on computer vision},
    pages={2425--2433},
    year={2015}
}

@article{liu-etal:2023visual,
    title={Visual instruction tuning},
    author={Liu, Haotian and Li, Chunyuan and Wu, Qingyang and Lee, Yong Jae},
    journal={arXiv preprint arXiv:2304.08485},
    year={2023}
}

@article{yin-etal:2023survey,
    title={A Survey on Multimodal Large Language Models},
    author={Yin, Shukang and Fu, Chaoyou and Zhao, Sirui and Li, Ke and Sun, Xing and Xu, Tong and Chen, Enhong},
    journal={arXiv preprint arXiv:2306.13549},
    year={2023}
}

@article{lin-etal:2022survey,
    title={A survey of transformers},
    author={Lin, Tianyang and Wang, Yuxin and Liu, Xiangyang and Qiu, Xipeng},
    journal={AI Open},
    year={2022},
    publisher={Elsevier}
}

@inproceedings{press-etal:2021train,
    title={Train Short, Test Long: Attention with Linear Biases Enables Input Length Extrapolation},
    author={Press, Ofir and Smith, Noah and Lewis, Mike},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2021}
}

@article{hestness-etal:2017deep,
    title={Deep learning scaling is predictable, empirically},
    author={Hestness, Joel and Narang, Sharan and Ardalani, Newsha and Diamos, Gregory and Jun, Heewoo and Kianinejad, Hassan and Patwary, Md Mostofa Ali and Yang, Yang and Zhou, Yanqi},
    journal={arXiv preprint arXiv:1712.00409},
    year={2017}
}

@article{bommasani-etal:2021foundation,
    title={On the Opportunities and Risks of Foundation Models},
    author={Rishi Bommasani and Drew A. Hudson and Ehsan Adeli and Russ Altman and Simran Arora and Sydney von Arx and Michael S. Bernstein and Jeannette Bohg and Antoine Bosselut and Emma Brunskill and Erik Brynjolfsson and S. Buch and Dallas Card and Rodrigo Castellon and Niladri S. Chatterji and Annie S. Chen and Kathleen A. Creel and Jared Davis and Dora Demszky and Chris Donahue and Moussa Doumbouya and Esin Durmus and Stefano Ermon and John Etchemendy and Kawin Ethayarajh and Li Fei-Fei and Chelsea Finn and Trevor Gale and Lauren E. Gillespie and Karan Goel and Noah D. Goodman and Shelby Grossman and Neel Guha and Tatsunori Hashimoto and Peter Henderson and John Hewitt and Daniel E. Ho and Jenny Hong and Kyle Hsu and Jing Huang and Thomas F. Icard and Saahil Jain and Dan Jurafsky and Pratyusha Kalluri and Siddharth Karamcheti and Geoff Keeling and Fereshte Khani and O. Khattab and Pang Wei Koh and Mark S. Krass and Ranjay Krishna and Rohith Kuditipudi and Ananya Kumar and Faisal Ladhak and Mina Lee and Tony Lee and Jure Leskovec and Isabelle Levent and Xiang Lisa Li and Xuechen Li and Tengyu Ma and Ali Malik and Christopher D. Manning and Suvir P. Mirchandani and Eric Mitchell and Zanele Munyikwa and Suraj Nair and Avanika Narayan and Deepak Narayanan and Benjamin Newman and Allen Nie and Juan Carlos Niebles and Hamed Nilforoshan and J. F. Nyarko and Giray Ogut and Laurel Orr and Isabel Papadimitriou and Joon Sung Park and Chris Piech and Eva Portelance and Christopher Potts and Aditi Raghunathan and Robert Reich and Hongyu Ren and Frieda Rong and Yusuf H. Roohani and Camilo Ruiz and Jack Ryan and Christopher R'e and Dorsa Sadigh and Shiori Sagawa and Keshav Santhanam and Andy Shih and Krishna Parasuram Srinivasan and Alex Tamkin and Rohan Taori and Armin W. Thomas and Florian Tram{\`e}r and Rose E. Wang and William Wang and Bohan Wu and Jiajun Wu and Yuhuai Wu and Sang Michael Xie and Michihiro Yasunaga and Jiaxuan You and Matei A. Zaharia and Michael Zhang and Tianyi Zhang and Xikun Zhang and Yuhui Zhang and Lucia Zheng and Kaitlyn Zhou and Percy Liang},
    journal={ArXiv},
    year={2021}
}

@article{bubeck-etal:2023sparks,
    title={Sparks of artificial general intelligence: Early experiments with gpt-4},
    author={S{\'{e}}bastien Bubeck and Varun Chandrasekaran and Ronen Eldan and Johannes Gehrke and Eric Horvitz and Ece Kamar and Peter Lee and Yin Tat Lee and Yuanzhi Li and Scott M. Lundberg and Harsha Nori and Hamid Palangi and Marco T{\'{u}}lio Ribeiro and Yi Zhang},
    journal={arXiv preprint arXiv:2303.12712},
    year={2023}
}

@article{yang-etal:2023dawn,
    title={The dawn of lmms: Preliminary explorations with gpt-4v (ision)},
    author={Yang, Zhengyuan and Li, Linjie and Lin, Kevin and Wang, Jianfeng and Lin, Chung-Ching and Liu, Zicheng and Wang, Lijuan},
    journal={arXiv preprint arXiv:2309.17421},
    year={2023}
}

@article{yu-etal:2023white,
    title={White-Box Transformers via Sparse Rate Reduction},
    author={Yu, Yaodong and Buchanan, Sam and Pai, Druv and Chu, Tianzhe and Wu, Ziyang and Tong, Shengbang and Haeffele, Benjamin D and Ma, Yi},
    journal={arXiv preprint arXiv:2306.01129},
    year={2023}
}

@inproceedings{li-etal:2022theoretical,
    title={A Theoretical Understanding of Shallow Vision Transformers: Learning, Generalization, and Sample Complexity},
    author={Li, Hongkang and Wang, Meng and Liu, Sijia and Chen, Pin-Yu},
    booktitle={The Eleventh International Conference on Learning Representations},
    year={2022}
}

@inproceedings{yun-etal:2019transformers,
    title={Are Transformers universal approximators of sequence-to-sequence functions?},
    author={Yun, Chulhee and Bhojanapalli, Srinadh and Rawat, Ankit Singh and Reddi, Sashank and Kumar, Sanjiv},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2019}
}

@inproceedings{perez-etal:2018turing,
    title={On the Turing Completeness of Modern Neural Network Architectures},
    author={P{\'e}rez, Jorge and Marinkovi{\'c}, Javier and Barcel{\'o}, Pablo},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2018}
}

@inproceedings{bhattamishra-etal:2020ability,
    title={On the Ability and Limitations of Transformers to Recognize Formal Languages},
    author={Bhattamishra, Satwik and Ahuja, Kabir and Goyal, Navin},
    booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
    pages={7096--7116},
    year={2020}
}

@article{hahn-etal:2020theoretical,
    title={Theoretical limitations of self-attention in neural sequence models},
    author={Hahn, Michael},
    journal={Transactions of the Association for Computational Linguistics},
    volume={8},
    pages={156--171},
    year={2020},
    publisher={MIT Press}
}

@article{hao-etal:2022formal,
    title={Formal language recognition by hard attention transformers: Perspectives from circuit complexity},
    author={Hao, Yiding and Angluin, Dana and Frank, Robert},
    journal={Transactions of the Association for Computational Linguistics},
    volume={10},
    pages={800--810},
    year={2022},
    publisher={MIT Press}
}

@article{merrill-etal:2022saturated,
    title={Saturated transformers are constant-depth threshold circuits},
    author={Merrill, William and Sabharwal, Ashish and Smith, Noah A},
    journal={Transactions of the Association for Computational Linguistics},
    volume={10},
    pages={843--856},
    year={2022},
    publisher={MIT Press}
}

@inproceedings{weiss-etal:2021thinking,
    title={Thinking like transformers},
    author={Weiss, Gail and Goldberg, Yoav and Yahav, Eran},
    booktitle={Proceedings of International Conference on Machine Learning},
    pages={11080--11090},
    year={2021},
    organization={PMLR}
}

@article{chiang-etal:2023tighter,
    title={Tighter Bounds on the Expressivity of Transformer Encoders},
    author={Chiang, David and Cholak, Peter and Pillay, Anand},
    journal={arXiv preprint arXiv:2301.10743},
    year={2023}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CHAPTER 8

@article{schmidhuber-etal:2015deep,
    title={Deep learning in neural networks: An overview},
    author={Schmidhuber, J{\"u}rgen},
    journal={Neural networks},
    volume={61},
    pages={85--117},
    year={2015},
    publisher={Elsevier}
}

@inproceedings{he-etal:2019rethinking,
    title={Rethinking imagenet pre-training},
    author={He, Kaiming and Girshick, Ross and Doll{\'a}r, Piotr},
    booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
    pages={4918--4927},
    year={2019}
}

@article{zoph-etal:2020rethinking,
    title={Rethinking pre-training and self-training},
    author={Zoph, Barret and Ghiasi, Golnaz and Lin, Tsung-Yi and Cui, Yin and Liu, Hanxiao and Cubuk, Ekin Dogus and Le, Quoc},
    journal={Advances in neural information processing systems},
    volume={33},
    pages={3833--3845},
    year={2020}
}

@inproceedings{yarowsky:1995unsupervised,
    title={Unsupervised word sense disambiguation rivaling supervised methods},
    author={Yarowsky, David},
    booktitle={Proceedings of the 33rd annual meeting of the association for computational linguistics},
    pages={189--196},
    year={1995}
}

@inproceedings{blum-and-tom:1998combining,
    title={Combining labeled and unlabeled data with co-training},
    author={Blum, Avrim and Mitchell, Tom},
    booktitle={Proceedings of the eleventh annual conference on Computational learning theory},
    pages={92--100},
    year={1998}
}

@article{yang-etal:2019xlnet,
    title={XLNet: Generalized autoregressive pretraining for language understanding},
    author={Yang, Zhilin and Dai, Zihang and Yang, Yiming and Carbonell, Jaime and Salakhutdinov, Russ R and Le, Quoc V},
    journal={Advances in neural information processing systems},
    volume={32},
    year={2019}
}

@inproceedings{clark-etal:2019electra,
    title={ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators},
    author={Clark, Kevin and Luong, Minh-Thang and Le, Quoc V and Manning, Christopher D},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2019}
}

@inproceedings{song-etal:2019mass,
    title={MASS: Masked Sequence to Sequence Pre-training for Language Generation},
    author={Song, Kaitao and Tan, Xu and Qin, Tao and Lu, Jianfeng and Liu, Tie-Yan},
    booktitle={International Conference on Machine Learning},
    pages={5926--5936},
    year={2019},
    organization={PMLR}
}

@article{joshi-etal:2020spanbert,
    title={SpanBERT: Improving pre-training by representing and predicting spans},
    author={Joshi, Mandar and Chen, Danqi and Liu, Yinhan and Weld, Daniel S and Zettlemoyer, Luke and Levy, Omer},
    journal={Transactions of the association for computational linguistics},
    volume={8},
    pages={64--77},
    year={2020},
    publisher={MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals-info~…}
}

@inproceedings{lewis-etal:2020bart,
    title={BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension},
    author={Lewis, Mike and Liu, Yinhan and Goyal, Naman and Ghazvininejad, Marjan and Mohamed, Abdelrahman and Levy, Omer and Stoyanov, Veselin and Zettlemoyer, Luke},
    booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
    pages={7871--7880},
    year={2020}
}

@article{dong-etal:2019unified,
    title={Unified language model pre-training for natural language understanding and generation},
    author={Dong, Li and Yang, Nan and Wang, Wenhui and Wei, Furu and Liu, Xiaodong and Wang, Yu and Gao, Jianfeng and Zhou, Ming and Hon, Hsiao-Wuen},
    journal={Advances in neural information processing systems},
    volume={32},
    year={2019}
}

@article{qiu-etal:2020pre,
    title={Pre-trained models for natural language processing: A survey},
    author={Qiu, Xipeng and Sun, Tianxiang and Xu, Yige and Shao, Yunfan and Dai, Ning and Huang, Xuanjing},
    journal={Science China Technological Sciences},
    volume={63},
    number={10},
    pages={1872--1897},
    year={2020},
    publisher={Springer}
}

@article{han-etal:2021pre,
    title={Pre-trained models: Past, present and future},
    author = {Xu Han and Zhengyan Zhang and Ning Ding and Yuxian Gu and Xiao Liu and Yuqi Huo and Jiezhong Qiu and Liang Zhang and Wentao Han and Minlie Huang and Qin Jin and Yanyan Lan and Yang Liu and Zhiyuan Liu and Zhiwu Lu and Xipeng Qiu and Ruihua Song and Jie Tang and Ji-Rong Wen and Jinhui Yuan and Wayne Xin Zhao and Jun Zhu},
    journal={AI Open},
    volume={2},
    pages={225--250},
    year={2021},
    publisher={Elsevier}
}

@article{liu-etal:2019roberta,
    title={Roberta: A robustly optimized bert pretraining approach},
    author={Liu, Yinhan and Ott, Myle and Goyal, Naman and Du, Jingfei and Joshi, Mandar and Chen, Danqi and Levy, Omer and Lewis, Mike and Zettlemoyer, Luke and Stoyanov, Veselin},
    journal={arXiv preprint arXiv:1907.11692},
    year={2019}
}

@article{shoeybi-etal:2019megatron,
    title={Megatron-lm: Training multi-billion parameter language models using model parallelism},
    author={Shoeybi, Mohammad and Patwary, Mostofa and Puri, Raul and LeGresley, Patrick and Casper, Jared and Catanzaro, Bryan},
    journal={arXiv preprint arXiv:1909.08053},
    year={2019}
}

@inproceedings{sun2020-etal:mobilebert,
    title={MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices},
    author={Sun, Zhiqing and Yu, Hongkun and Song, Xiaodan and Liu, Renjie and Yang, Yiming and Zhou, Denny},
    booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
    pages={2158--2170},
    year={2020}
}

@inproceedings{jiao-etal:2020tinybert,
    title={TinyBERT: Distilling BERT for Natural Language Understanding},
    author={Jiao, Xiaoqi and Yin, Yichun and Shang, Lifeng and Jiang, Xin and Chen, Xiao and Li, Linlin and Wang, Fang and Liu, Qun},
    booktitle={Findings of the Association for Computational Linguistics: EMNLP 2020},
    pages={4163--4174},
    year={2020}
}

@article{gale-etal:2019state,
    title={The state of sparsity in deep neural networks},
    author={Gale, Trevor and Elsen, Erich and Hooker, Sara},
    journal={arXiv preprint arXiv:1902.09574},
    year={2019}
}

@article{sanh-etal:2020movement,
    title={Movement pruning: Adaptive sparsity by fine-tuning},
    author={Sanh, Victor and Wolf, Thomas and Rush, Alexander},
    journal={Advances in Neural Information Processing Systems},
    volume={33},
    pages={20378--20389},
    year={2020}
}

@article{chen-etal:2020lottery,
    title={The lottery ticket hypothesis for pre-trained bert networks},
    author={Chen, Tianlong and Frankle, Jonathan and Chang, Shiyu and Liu, Sijia and Zhang, Yang and Wang, Zhangyang and Carbin, Michael},
    journal={Advances in neural information processing systems},
    volume={33},
    pages={15834--15846},
    year={2020}
}

@inproceedings{fan-etal:2019reducing,
    title={Reducing Transformer Depth on Demand with Structured Dropout},
    author={Fan, Angela and Grave, Edouard and Joulin, Armand},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2019}
}

@inproceedings{shen-etal:2020qbert,
    title={Q-bert: Hessian based ultra low precision quantization of bert},
    author={Shen, Sheng and Dong, Zhen and Ye, Jiayu and Ma, Linjian and Yao, Zhewei and Gholami, Amir and Mahoney, Michael W and Keutzer, Kurt},
    booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
    volume={34},
    pages={8815--8821},
    year={2020}
}

@inproceedings{lan-etal:2020albert,
    title={ALBERT: A Lite BERT for Self-supervised Learning of Language Representations},
    author={Lan, Zhenzhong and Chen, Mingda and Goodman, Sebastian and Gimpel, Kevin and Sharma, Piyush and Soricut, Radu},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2020}
}

@inproceedings{conneau-etal:2020unsupervised,
    title={Unsupervised Cross-lingual Representation Learning at Scale},
    author={Conneau, Alexis and Khandelwal, Kartikay and Goyal, Naman and Chaudhary, Vishrav and Wenzek, Guillaume and Guzm{\'a}n, Francisco and Grave, {\'E}douard and Ott, Myle and Zettlemoyer, Luke and Stoyanov, Veselin},
    booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
    pages={8440--8451},
    year={2020}
}

@article{warstadt-etal:2019neural,
    title={Neural network acceptability judgments},
    author={Warstadt, Alex and Singh, Amanpreet and Bowman, Samuel R},
    journal={Transactions of the Association for Computational Linguistics},
    volume={7},
    pages={625--641},
    year={2019},
    publisher={MIT Press}
}

@inproceedings{socher-etal:2013recursive,
    title={Recursive deep models for semantic compositionality over a sentiment treebank},
    author={Socher, Richard and Perelygin, Alex and Wu, Jean and Chuang, Jason and Manning, Christopher D and Ng, Andrew Y and Potts, Christopher},
    booktitle={Proceedings of the 2013 conference on empirical methods in natural language processing},
    pages={1631--1642},
    year={2013}
}

@inproceedings{dolan-brockett:2005automatically,
    title={Automatically constructing a corpus of sentential paraphrases},
    author={Dolan, Bill and Brockett, Chris},
    booktitle={Proceedings of Third International Workshop on Paraphrasing (IWP2005)},
    year={2005}
}

@inproceedings{williams-etal:2018broad,
    title={A Broad-Coverage Challenge Corpus for Sentence Understanding through Inference},
    author={Williams, Adina and Nangia, Nikita and Bowman, Samuel},
    booktitle={Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)},
    pages={1112--1122},
    year={2018}
}

@misc{bentivogli-giampiccolo:2011rte,
    howpublished = {\url{https://tac.nist.gov/2011/RTE/}},
    title = {PASCAL Recognizing Textual Entailment Challenge (RTE-7) at TAC 2011},
    author = {Luisa Bentivogli and Danilo Giampiccolo},
    year = {2011}
}

@inproceedings{zellers-etal:2018swag,
    title={SWAG: A Large-Scale Adversarial Dataset for Grounded Commonsense Inference},
    author={Zellers, Rowan and Bisk, Yonatan and Schwartz, Roy and Choi, Yejin},
    booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
    pages={93--104},
    year={2018}
}

@inproceedings{huang-etal:2009dynamic,
    title = "Dynamic Programming-based Search Algorithms in {NLP}",
    author = "Huang, Liang",
    booktitle = "Proceedings of Human Language Technologies: The 2009 Annual Conference of the North {A}merican Chapter of the Association for Computational Linguistics, Companion Volume: Tutorial Abstracts",
    year = "2009",
}

@inproceedings{seo-etal::2017bidirectional,
    title={Bidirectional Attention Flow for Machine Comprehension},
    author={Seo, Minjoon and Kembhavi, Aniruddha and Farhadi, Ali and Hajishirzi, Hannaneh},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2017}
}

@article{rolnick-etal:2019experience,
    title={Experience replay for continual learning},
    author={Rolnick, David and Ahuja, Arun and Schwarz, Jonathan and Lillicrap, Timothy and Wayne, Gregory},
    journal={Advances in Neural Information Processing Systems},
    volume={32},
    year={2019}
}

@article{kirkpatrick-etal:2017overcoming,
    title={Overcoming catastrophic forgetting in neural networks},
    author = {Kirkpatrick, James and Pascanu, Razvan and Rabinowitz, Neil and Veness, Joel and Desjardins, Guillaume and Rusu, Andrei A. and Milan, Kieran and Quan, John and Ramalho, Tiago and Grabska-Barwinska, Agnieszka and Hassabis, Demis and Clopath, Claudia and Kumaran, Dharshan and Hadsell, Raia},
    journal={Proceedings of the national academy of sciences},
    volume={114},
    number={13},
    pages={3521--3526},
    year={2017},
    publisher={National Acad Sciences}
}

@article{parisi-etal:2019continual,
    title={Continual lifelong learning with neural networks: A review},
    author={Parisi, German I and Kemker, Ronald and Part, Jose L and Kanan, Christopher and Wermter, Stefan},
    journal={Neural networks},
    volume={113},
    pages={54--71},
    year={2019},
    publisher={Elsevier}
}

@article{wang-etal:2023comprehensive,
    title={A comprehensive survey of continual learning: Theory, method and application},
    author={Wang, Liyuan and Zhang, Xingxing and Su, Hang and Zhu, Jun},
    journal={arXiv preprint arXiv:2302.00487},
    year={2023}
}

@article{wang-etal:2023comprehensive2,
    title={A comprehensive survey of forgetting in deep learning beyond continual learning},
    author={Wang, Zhenyi and Yang, Enneng and Shen, Li and Huang, Heng},
    journal={arXiv preprint arXiv:2307.09218},
    year={2023}
}

@article{manning:2022human,
    title={Human language understanding \& reasoning},
    author={Manning, Christopher D},
    journal={Daedalus},
    volume={151},
    number={2},
    pages={127--138},
    year={2022},
    publisher={MIT Press}
}

@article{shannon:1951prediction,
    title={Prediction and entropy of printed English},
    author={Shannon, Claude E},
    journal={Bell system technical journal},
    volume={30},
    number={1},
    pages={50--64},
    year={1951},
    publisher={Wiley Online Library}
}

@book{jelinek:1998statistical,
    title={Statistical methods for speech recognition},
    author={Jelinek, Frederick},
    year={1998},
    publisher={MIT Press}
}

@article{radford-etal:2019language,
    title={Language models are unsupervised multitask learners},
    author={Radford, Alec and Wu, Jeffrey and Child, Rewon and Luan, David and
            Amodei, Dario and Sutskever, Ilya},
    journal={OpenAI blog},
    volume={1},
    number={8},
    year={2019},
    publisher={Open AI}
}

@article{penedo-etal:2023refinedweb,
    title={The RefinedWeb dataset for Falcon LLM: outperforming curated corpora with web data, and web data only},
    author={Penedo, Guilherme and Malartic, Quentin and Hesslow, Daniel and Cojocaru, Ruxandra and Cappelli, Alessandro and Alobeidli, Hamza and Pannier, Baptiste and Almazrouei, Ebtesam and Launay, Julien},
    journal={arXiv preprint arXiv:2306.01116},
    year={2023}
}

@article{jiang-etal:2023mistral,
    title={Mistral 7B},
    author={Jiang, Albert Q and Sablayrolles, Alexandre and Mensch, Arthur and Bamford, Chris and Chaplot, Devendra Singh and Casas, Diego de las and Bressand, Florian and Lengyel, Gianna and Lample, Guillaume and Saulnier, Lucile and Lavaud, L{\'e}lio Renard and Lachaux, Marie-Anne and Stock, Pierre and Le Scao, Teven and Lavril, Thibaut and Wang, Thomas and Lacroix, Timoth{\'e}e and El Sayed, William},
    journal={arXiv preprint arXiv:2310.06825},
    year={2023}
}

@article{chung-etal:2022scaling,
  title={Scaling instruction-finetuned language models},
  author={Chung, Hyung~Won and Hou, Le~ and Longpre, S. and Zoph, Barret and Tay, Yi and Fedus, William and Li, Eric and Wang, Xuezhi and Dehghani, Mostafa and Brahma, Siddhartha and Webson, Albert and Gu, Shixiang~Shane and Dai, Zhuyun and Suzgun, Mirac and Chen, Xinyun and Chowdhery, Aakanksha and Valter, Dasha and Narang, Sharan and Mishra, Gaurav and Yu, Adams~Wei and Zhao, Vincent and Huang, Yanping and Dai, Andrew~M. and Yu, Hongkun and Petrov, Slav and Chi, Ed~Huai hsin and Dean, Jeff and Devlin, Jacob and Roberts, Adam and Zhou, Denny and Le, Quoc~V. and Wei, Jason},
  journal={arXiv preprint arXiv:2210.11416},
  year={2022}
}

@inproceedings{sanh-etal:2022multitask,
    title={Multitask Prompted Training Enables Zero-Shot Task Generalization},
    author = {Victor Sanh and Albert Webson and Colin Raffel and Stephen Bach and Lintang Sutawika and Zaid Alyafeai and Antoine Chaffin and Arnaud Stiegler and Arun Raja and Manan Dey and M Saiful Bari and Canwen Xu and Urmish Thakker and Shanya Sharma Sharma and Eliza Szczechla and Taewoon Kim and Gunjan Chhablani and Nihal Nayak and Debajyoti Datta and Jonathan Chang and Mike Tian-Jian Jiang and Han Wang and Matteo Manica and Sheng Shen and Zheng Xin Yong and Harshit Pandey and Rachel Bawden and Thomas Wang and Trishala Neeraj and Jos Rozen and Abheesht Sharma and Andrea Santilli and Thibault Fevry and Jason Alan Fries and Ryan Teehan and Teven Le Scao and Stella Biderman and Leo Gao and Thomas Wolf and Alexander M Rush},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2022}
}

@inproceedings{wei-etal:2022finetuned,
    title={Finetuned Language Models are Zero-Shot Learners},
    author={Wei, Jason and Bosma, Maarten and Zhao, Vincent and Guu, Kelvin and Yu, Adams Wei and Lester, Brian and Du, Nan and Dai, Andrew M and Le, Quoc V},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2022}
}

@article{zhou-etal:2023lima,
    title={Lima: Less is more for alignment},
    author={Zhou, Chunting and Liu, Pengfei and Xu, Puxin and Iyer, Srini and Sun, Jiao and Mao, Yuning and Ma, Xuezhe and Efrat, Avia and Yu, Ping and Yu, Lili and Zhang, Susan and Ghosh, Gargi and Lewis, Mike and Zettlemoyer, Luke and Levy, Omer},
    journal={arXiv preprint arXiv:2305.11206},
    year={2023}
}

@article{chen-etal:2023alpagasus,
    title={Alpagasus: Training a better alpaca with fewer data},
    author={Chen, Lichang and Li, Shiyang and Yan, Jun and Wang, Hai and Gunaratna, Kalpa and Yadav, Vikas and Tang, Zheng and Srinivasan, Vijay and Zhou, Tianyi and Huang, Heng and Jin, Hongxia},
    journal={arXiv preprint arXiv:2307.08701},
    year={2023}
}

@article{christiano-etal:2017deep,
    title={Deep reinforcement learning from human preferences},
    author={Christiano, Paul F and Leike, Jan and Brown, Tom and Martic, Miljan and Legg, Shane and Amodei, Dario},
    journal={Advances in neural information processing systems},
    volume={30},
    year={2017}
}

@article{stiennon-etal:2020learning,
    title={Learning to summarize with human feedback},
    author={Stiennon, Nisan and Ouyang, Long and Wu, Jeffrey and Ziegler, Daniel and Lowe, Ryan and Voss, Chelsea and Radford, Alec and Amodei, Dario and Christiano, Paul F},
    journal={Advances in Neural Information Processing Systems},
    volume={33},
    pages={3008--3021},
    year={2020}
}

@article{wei-etal:2022chain,
    title={Chain-of-thought prompting elicits reasoning in large language models},
    author={Jason Wei and Xuezhi Wang and Dale Schuurmans and Maarten Bosma and Brian Ichter and Fei Xia and Ed H. Chi and Quoc V. Le and Denny Zhou},
    journal={Advances in Neural Information Processing Systems},
    volume={35},
    pages={24824--24837},
    year={2022}
}

@article{kojima-etal:2022large,
    title={Large language models are zero-shot reasoners},
    author={Kojima, Takeshi and Gu, Shixiang Shane and Reid, Machel and Matsuo, Yutaka and Iwasawa, Yusuke},
    journal={Advances in neural information processing systems},
    volume={35},
    pages={22199--22213},
    year={2022}
}

@article{almazrouei-etal:2023falcon,
    title={The falcon series of open language models},
    author={Ebtesam Almazrouei and
                  Hamza Alobeidli and
                  Abdulaziz Alshamsi and
                  Alessandro Cappelli and
                  Ruxandra Cojocaru and
                  M{\'{e}}rouane Debbah and
                  {\'{E}}tienne Goffinet and
                  Daniel Hesslow and
                  Julien Launay and
                  Quentin Malartic and
                  Daniele Mazzotta and
                  Badreddine Noune and
                  Baptiste Pannier and
                  Guilherme Penedo},
    journal={arXiv preprint arXiv:2311.16867},
    year={2023}
}

@misc{gemma:24,
  title={{Gemma: Open Models Based on Gemini Research and Technology}},
  author={Gemma Team, Google DeepMind},
  year={2024}
}

@article{team2024gemma,
  title={Gemma 2: Improving open language models at a practical size},
  author={Team, Gemma and Riviere, Morgane and Pathak, Shreya and Sessa, Pier Giuseppe and Hardin, Cassidy and Bhupatiraju, Surya and Hussenot, L{\'e}onard and Mesnard, Thomas and Shahriari, Bobak and Ram{\'e}, Alexandre and others},
  journal={arXiv preprint arXiv:2408.00118},
  year={2024}
}

@article{zhang-sennrich:2019root,
    title={Root mean square layer normalization},
    author={Zhang, Biao and Sennrich, Rico},
    journal={Advances in Neural Information Processing Systems},
    volume={32},
    year={2019}
}

@inproceedings{dong-etal:2021attention,
    title={Attention is not all you need: Pure attention loses rank doubly exponentially with depth},
    author={Dong, Yihe and Cordonnier, Jean-Baptiste and Loukas, Andreas},
    booktitle={International Conference on Machine Learning},
    pages={2793--2803},
    year={2021},
    organization={PMLR}
}

@article{hendrycks-gimpel:2016gaussian,
    title={Gaussian error linear units (gelus)},
    author={Hendrycks, Dan and Gimpel, Kevin},
    journal={arXiv preprint arXiv:1606.08415},
    year={2016}
}

@article{ramachandran-etal:2017searching,
    title={Searching for activation functions},
    author={Ramachandran, Prajit and Zoph, Barret and Le, Quoc V},
    journal={arXiv preprint arXiv:1710.05941},
    year={2017}
}

@article{shazeer-etal:2020glu,
    title={Glu variants improve transformer},
    author={Shazeer, Noam},
    journal={arXiv preprint arXiv:2002.05202},
    year={2020}
}

@inproceedings{narayanan-etal:2021efficient,
    title={Efficient large-scale language model training on gpu clusters using megatron-lm},
    author={Deepak Narayanan and Mohammad Shoeybi and Jared Casper and Patrick LeGresley and Mostofa Patwary and Vijay Korthikanti and Dmitri Vainbrand and Prethvi Kashinkunti and Julie Bernauer and Bryan Catanzaro and Amar Phanishayee and
    Matei Zaharia},
    booktitle={Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis},
    pages={1--15},
    year={2021}
}

@article{harlap-etal:2018pipedream,
  title={Pipedream: Fast and efficient pipeline parallel dnn training},
  author={Harlap, Aaron and Narayanan, Deepak and Phanishayee, Amar and Seshadri, Vivek and Devanur, Nikhil and Ganger, Greg and Gibbons, Phil},
  journal={arXiv preprint arXiv:1806.03377},
  year={2018}
}

@article{huang-etal:2019gpipe,
    title={Gpipe: Efficient training of giant neural networks using pipeline parallelism},
    author={Huang, Yanping and Cheng, Youlong and Bapna, Ankur and Firat, Orhan and Chen, Mia Xu and Chen, Dehao and Lee, HyoukJoong and Ngiam, Jiquan and Le, Quoc V and Wu, Yonghui and Chen, Zhifeng},
    journal={Advances in neural information processing systems},
    volume={32},
    year={2019}
}

@article{weng:2021large,
    title   = "How to Train Really Large Models on Many GPUs?",
    author  = "Weng, Lilian",
    journal = "lilianweng.github.io",
    year    = "2021",
    month   = "Sep",
    url     = "https://lilianweng.github.io/posts/2021-09-25-train-large/"
}

@inproceedings{micikevicius-etal:2018mixed,
    title={Mixed Precision Training},
    author={Micikevicius, Paulius and Narang, Sharan and Alben, Jonah and Diamos, Gregory and Elsen, Erich and Garcia, David and Ginsburg, Boris and Houston, Michael and Kuchaiev, Oleksii and Venkatesh, Ganesh and Wu, Hao},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2018}
}

@inproceedings{gordon-etal:2021data,
    title={Data and parameter scaling laws for neural machine translation},
    author={Gordon, Mitchell A and Duh, Kevin and Kaplan, Jared},
    booktitle={Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},
    pages={5915--5922},
    year={2021}
}

@inproceedings{rosenfeld-etal:2020constructive,
    title={A Constructive Prediction of the Generalization Error Across Scales},
    author={Rosenfeld, Jonathan S and Rosenfeld, Amir and Belinkov, Yonatan and Shavit, Nir},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2020}
}

@article{hoffmann-etal:2022training,
    title={Training compute-optimal large language models},
    author={Jordan Hoffmann and
                  Sebastian Borgeaud and
                  Arthur Mensch and
                  Elena Buchatskaya and
                  Trevor Cai and
                  Eliza Rutherford and
                  Diego de Las Casas and
                  Lisa Anne Hendricks and
                  Johannes Welbl and
                  Aidan Clark and
                  Tom Hennigan and
                  Eric Noland and
                  Katie Millican and
                  George van den Driessche and
                  Bogdan Damoc and
                  Aurelia Guy and
                  Simon Osindero and
                  Karen Simonyan and
                  Erich Elsen and
                  Jack W. Rae and
                  Oriol Vinyals and
                  Laurent Sifre},
    journal={arXiv preprint arXiv:2203.15556},
    year={2022}
}

@article{alabdulmohsin-etal:2022revisiting,
    title={Revisiting neural scaling laws in language and vision},
    author={Alabdulmohsin, Ibrahim M and Neyshabur, Behnam and Zhai, Xiaohua},
    journal={Advances in Neural Information Processing Systems},
    volume={35},
    pages={22300--22312},
    year={2022}
}

@inproceedings{caballero-etal:2023broken,
    title={Broken Neural Scaling Laws},
    author={Caballero, Ethan and Gupta, Kshitij and Rish, Irina and Krueger, David},
    booktitle={ICLR 2023 Workshop on Mathematical and Empirical Understanding of Foundation Models},
    year={2023}
}

@article{gu-and-dao:2023mamba,
    title={Mamba: Linear-time sequence modeling with selective state spaces},
    author={Gu, Albert and Dao, Tri},
    journal={arXiv preprint arXiv:2312.00752},
    year={2023}
}

@article{hutchins-etal:2022block,
    title={Block-recurrent transformers},
    author={Hutchins, DeLesley and Schlag, Imanol and Wu, Yuhuai and Dyer, Ethan and Neyshabur, Behnam},
    journal={Advances in neural information processing systems},
    volume={35},
    pages={33248--33261},
    year={2022}
}

@article{bulatov-etal:2022recurrent,
    title={Recurrent memory transformer},
    author={Bulatov, Aydar and Kuratov, Yury and Burtsev, Mikhail},
    journal={Advances in Neural Information Processing Systems},
    volume={35},
    pages={11079--11091},
    year={2022}
}

@inproceedings{rae-etal:2020:compressive,
    title={Compressive Transformers for Long-Range Sequence Modelling},
    author={Rae, Jack W and Potapenko, Anna and Jayakumar, Siddhant M and Hillier, Chloe and Lillicrap, Timothy P},
    booktitle={International Conference on Learning Representations},
    year={2019}
}

@article{gupta-berant:2020gmat,
    title={Gmat: Global memory augmentation for transformers},
    author={Gupta, Ankit and Berant, Jonathan},
    journal={arXiv preprint arXiv:2006.03274},
    year={2020}
}

@inproceedings{xiao-etal:2024efficient,
    title={Efficient Streaming Language Models with Attention Sinks},
    author={Guangxuan Xiao and Yuandong Tian and Beidi Chen and Song Han and Mike Lewis},
    booktitle={Proceedings of The Twelfth International Conference on Learning Representations},
    year={2024}
}

@inproceedings{gupta-etal:2021memory,
    title={Memory-efficient Transformers via Top-k Attention},
    author={Gupta, Ankit and Dar, Guy and Goodman, Shaya and Ciprut, David and Berant, Jonathan},
    booktitle={Proceedings of the Second Workshop on Simple and Efficient Natural Language Processing},
    pages={39--52},
    year={2021}
}

@inproceedings{khandelwal-etal:2020generalization,
    title={Generalization through Memorization: Nearest Neighbor Language Models},
    author={Urvashi Khandelwal and Omer Levy and Dan Jurafsky and Luke Zettlemoyer and Mike Lewis},
    booktitle={International Conference on Learning Representations},
    year={2020}
}

@inproceedings{ma-etal:2023mega,
    title={Mega: Moving Average Equipped Gated Attention},
    author={Xuezhe Ma and Chunting Zhou and Xiang Kong and Junxian He and Liangke Gui and Graham Neubig and Jonathan May and Luke Zettlemoyer},
    booktitle={The Eleventh International Conference on Learning Representations },
    year={2023}
}

@article{ma-etal:2024megalodon,
    title={Megalodon: Efficient LLM Pretraining and Inference with Unlimited Context Length},
    author={Ma, Xuezhe and Yang, Xiaomeng and Xiong, Wenhan and Chen, Beidi and Yu, Lili and Zhang, Hao and May, Jonathan and Zettlemoyer, Luke and Levy, Omer and Zhou, Chunting},
    journal={arXiv preprint arXiv:2404.08801},
    year={2024}
}

@article{munkhdalai-etal:2024leave,
    title={Leave No Context Behind: Efficient Infinite Context Transformers with Infini-attention},
    author={Munkhdalai, Tsendsuren and Faruqui, Manaal and Gopal, Siddharth},
    journal={arXiv preprint arXiv:2404.07143},
    year={2024}
}

@inproceedings{li-etal:2023sequence,
    title={Sequence Parallelism: Long Sequence Training from System Perspective},
    author={Li, Shenggui and Xue, Fuzhao and Baranwal, Chaitanya and Li, Yongbin and You, Yang},
    booktitle={Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={2391--2404},
    year={2023}
}

@article{korthikanti-etal:2023reducing,
    title={Reducing activation recomputation in large transformer models},
    author={Korthikanti, Vijay Anand and Casper, Jared and Lym, Sangkug and McAfee, Lawrence and Andersch, Michael and Shoeybi, Mohammad and Catanzaro, Bryan},
    journal={Proceedings of Machine Learning and Systems},
    volume={5},
    year={2023}
}

@inproceedings{press-etal:2022train,
   title={Train Short, Test Long: Attention with Linear Biases Enables Input Length Extrapolation},
   author={Ofir Press and Noah Smith and Mike Lewis},
   booktitle={Proceedings of International Conference on Learning Representations},
   year={2022}
}

@article{chi-etal:2022kerple,
    title={Kerple: Kernelized relative positional embedding for length extrapolation},
    author={Chi, Ta-Chung and Fan, Ting-Han and Ramadge, Peter J and Rudnicky, Alexander},
    journal={Advances in Neural Information Processing Systems},
    volume={35},
    pages={8386--8399},
    year={2022}
}

@inproceedings{li-etal:2024functional,
    title={Functional Interpolation for Relative Positions improves Long Context Transformers},
    author={Shanda Li and Chong You and Guru Guruganesh and Joshua Ainslie and Santiago Ontanon and Manzil Zaheer and Sumit Sanghai and Yiming Yang and Sanjiv Kumar and Srinadh Bhojanapalli},
    booktitle={The Twelfth International Conference on Learning Representations},
    year={2024}
}

@inproceedings{chi-etal:2023dissecting,
    title={Dissecting Transformer Length Extrapolation via the Lens of Receptive Field Analysis},
    author={Chi, Ta-Chung and Fan, Ting-Han and Rudnicky, Alexander and Ramadge, Peter},
    booktitle={Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={13522--13537},
    year={2023}
}

@article{su-etal:2024roformer,
    title={Roformer: Enhanced transformer with rotary position embedding},
    author={Su, Jianlin and Ahmed, Murtadha and Lu, Yu and Pan, Shengfeng and Bo, Wen and Liu, Yunfeng},
    journal={Neurocomputing},
    volume={568},
    pages={127063},
    year={2024},
    publisher={Elsevier}
}

@Misc{biderman-etal:2021rotary,
    howpublished = {\url{https://blog.eleuther.ai/rotary-embeddings/}},
    title = {Rotary Embeddings: A Relative Revolution},
    author = {Stella Biderman and Sid Black and Charles Foster and Leo Gao and Eric Hallahan and Horace He and Ben Wang and Phil Wang},
    year = {2021}
}

@article{chen-etal:2023extending,
    title={Extending context window of large language models via positional interpolation},
    author={Chen, Shouyuan and Wong, Sherman and Chen, Liangjian and Tian, Yuandong},
    journal={arXiv preprint arXiv:2306.15595},
    year={2023}
}

@inproceedings{peng-etal:2024yarn,
    title={Ya{RN}: Efficient Context Window Extension of Large Language Models},
    author={Bowen Peng and Jeffrey Quesnelle and Honglu Fan and Enrico Shippole},
    booktitle={The Twelfth International Conference on Learning Representations},
    year={2024}
}

@article{ding-etal:2024longrope,
    title={LongRoPE: Extending LLM Context Window Beyond 2 Million Tokens},
    author={Ding, Yiran and Zhang, Li Lyna and Zhang, Chengruidong and Xu, Yuanyuan and Shang, Ning and Xu, Jiahang and Yang, Fan and Yang, Mao},
    journal={arXiv preprint arXiv:2402.13753},
    year={2024}
}

@inproceedings{deletang-etal:2024language,
    title={Language Modeling Is Compression},
    author={Gregoire Deletang and Anian Ruoss and Paul-Ambroise Duquenne and Elliot Catt and Tim Genewein and Christopher Mattern and Jordi Grau-Moya and Li Kevin Wenliang and Matthew Aitchison and Laurent Orseau and Marcus Hutter and Joel Veness},
    booktitle={The Twelfth International Conference on Learning Representations},
    year={2024}
}

@inproceedings{pal-etal:2023future,
    title={Future Lens: Anticipating Subsequent Tokens from a Single Hidden State},
    author={Pal, Koyena and Sun, Jiuding and Yuan, Andrew and Wallace, Byron C and Bau, David},
    booktitle={Proceedings of the 27th Conference on Computational Natural Language Learning (CoNLL)},
    pages={548--560},
    year={2023}
}

@article{wu-etal:2024language,
    title={Do Language Models Plan for Future Tokens?},
    author={Wu, Wilson and Morris, John X and Levine, Lionel},
    journal={arXiv preprint arXiv:2404.00859},
    year={2024}
}

@inproceedings{liu-etal:2024forgetting,
    title={Forgetting Curve: A Reliable Method for Evaluating Memorization Capability for Long-Context Models},
    author={Liu, Xinyu and Zhao, Runsong and Huang, Pengcheng and Xiao, Chunyang and Li, Bei and Wang, Jingang and Xiao, Tong and Zhu, Jingbo},
    booktitle={Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing},
    pages={4667--4682},
    year={2024}
}

@article{mohtashami-and-jaggi:2024random,
    title={Random-access infinite context length for transformers},
    author={Mohtashami, Amirkeivan and Jaggi, Martin},
    journal={Advances in Neural Information Processing Systems},
    volume={36},
    year={2024}
}

@inproceedings{arjovsky-etal:2016unitary,
    title={Unitary evolution recurrent neural networks},
    author={Arjovsky, Martin and Shah, Amar and Bengio, Yoshua},
    booktitle={International conference on machine learning},
    pages={1120--1128},
    year={2016}
}

@inproceedings{ainslie-etal:2023gqa,
    title={GQA: Training Generalized Multi-Query Transformer Models from Multi-Head Checkpoints},
    author={Ainslie, Joshua and Lee-Thorp, James and de Jong, Michiel and Zemlyanskiy, Yury and Lebron, Federico and Sanghai, Sumit},
    booktitle={Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},
    pages={4895--4901},
    year={2023}
}

@article{brandon-etal:2024reducing,
    title={Reducing Transformer Key-Value Cache Size with Cross-Layer Attention},
    author={Brandon, William and Mishra, Mayank and Nrusimha, Aniruddha and Panda, Rameswar and Kelly, Jonathan Ragan},
    journal={arXiv preprint arXiv:2405.12981},
    year={2024}
}

@misc{openai:2024learning,
    title = {Learning to Reason with LLMs},
    url = {https://openai.com/index/learning-to-reason-with-llms/},
    author = {OpenAI},
    month = {September},
    year = {2024}
}

@article{zhao-etal:2023survey,
    title={A survey of large language models},
    author={Wayne Xin Zhao and Kun Zhou and Junyi Li and Tianyi Tang and Xiaolei Wang and Yupeng Hou and Yingqian Min and Beichen Zhang and Junjie Zhang and Zican Dong and Yifan Du and Chen Yang and Yushuo Chen and Z. Chen and Jinhao Jiang and Ruiyang Ren and Yifan Li and Xinyu Tang and Zikang Liu and Peiyu Liu and Jianyun Nie and Ji-rong Wen},
    journal={arXiv preprint arXiv:2303.18223},
    year={2023}
}

@article{minaee-etal:2024large,
    title={Large language models: A survey},
    author={Minaee, Shervin and Mikolov, Tomas and Nikzad, Narjes and Chenaghlu, Meysam and Socher, Richard and Amatriain, Xavier and Gao, Jianfeng},
    journal={arXiv preprint arXiv:2402.06196},
    year={2024}
}

@misc{ruan-etal:2024abigsurveyofllms,
    howpublished={\url{https://github.com/NiuTrans/ABigSurveyOfLLMs}},
    title={A Survey of LLM Surveys},
    author={Junhao Ruan and Long Meng and Weiqiao Shan and Tong Xiao and Jingbo Zhu},
    year={2024}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CHAPTER 10

@article{liu-etal:2023pre,
    title={Pre-train, prompt, and predict: A systematic survey of prompting methods in natural language processing},
    author={Liu, Pengfei and Yuan, Weizhe and Fu, Jinlan and Jiang, Zhengbao and Hayashi, Hiroaki and Neubig, Graham},
    journal={ACM Computing Surveys},
    volume={55},
    number={9},
    pages={1--35},
    year={2023},
    publisher={ACM}
}

@inproceedings{xie-etal:2022an,
    title={An Explanation of In-context Learning as Implicit Bayesian Inference},
    author={Sang Michael Xie and Aditi Raghunathan and Percy Liang and Tengyu Ma},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2022}
}

@inproceedings{dai-etal:2023can,
    title={Why Can GPT Learn In-Context? Language Models Secretly Perform Gradient Descent as Meta-Optimizers},
    author={Dai, Damai and Sun, Yutao and Dong, Li and Hao, Yaru and Ma, Shuming and Sui, Zhifang and Wei, Furu},
    booktitle={Findings of the Association for Computational Linguistics: ACL 2023},
    pages={4005--4019},
    year={2023}
}

@inproceedings{von-etal:2023transformers,
    title={Transformers learn in-context by gradient descent},
    author={Von Oswald, Johannes and Niklasson, Eyvind and Randazzo, Ettore and Sacramento, Jo{\~a}o and Mordvintsev, Alexander and Zhmoginov, Andrey and Vladymyrov, Max},
    booktitle={Proceedings of International Conference on Machine Learning},
    pages={35151--35174},
    year={2023},
    organization={PMLR}
}

@inproceedings{akyurek-etal:2023what,
    title={What learning algorithm is in-context learning? Investigations with linear models},
    author={Ekin Aky{\"u}rek and Dale Schuurmans and Jacob Andreas and Tengyu Ma and Denny Zhou},
    booktitle={Proceedings of The Eleventh International Conference on Learning Representations},
    year={2023}
}

@article{garg-etal:2022can,
    title={What can transformers learn in-context? a case study of simple function classes},
    author={Garg, Shivam and Tsipras, Dimitris and Liang, Percy S and Valiant, Gregory},
    journal={Advances in Neural Information Processing Systems},
    volume={35},
    pages={30583--30598},
    year={2022}
}

@inproceedings{hendrycks-etal:2021measuring,
    title={Measuring Massive Multitask Language Understanding},
    author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
    booktitle={Proceedings of International Conference on Learning Representations},
    year={2021}
}

@article{cobbe-etal:2021training,
    title={Training verifiers to solve math word problems},
    author={Karl Cobbe and Vineet Kosaraju and Mohammad Bavarian and Mark Chen and Heewoo Jun and Lukasz Kaiser and Matthias Plappert and Jerry Tworek and Jacob Hilton and Reiichiro Nakano and Christopher Hesse and John Schulman},
    journal={arXiv preprint arXiv:2110.14168},
    year={2021}
}

@article{yao-etal:2024tree,
    title={Tree of thoughts: Deliberate problem solving with large language models},
    author={Yao, Shunyu and Yu, Dian and Zhao, Jeffrey and Shafran, Izhak and Griffiths, Tom and Cao, Yuan and Narasimhan, Karthik},
    journal={Advances in Neural Information Processing Systems},
    volume={36},
    year={2024}
}

@inproceedings{besta-etal:2024graph,
    title={Graph of thoughts: Solving elaborate problems with large language models},
    author={Maciej Besta and Nils Blach and Ales Kubicek and Robert Gerstenberger and Michal Podstawski and Lukas Gianinazzi and Joanna Gajda and Tomasz Lehmann and Hubert Niewiadomski and Piotr Nyczyk and Torsten Hoefler},
    booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
    volume={38},
    pages={17682--17690},
    year={2024}
}

@book{kahneman:2011thinking,
    title={Thinking, fast and slow},
    author={Kahneman, Daniel},
    year={2011},
    publisher={macmillan}
}

@article{chu-etal:2023survey,
    title={A survey of chain of thought reasoning: Advances, frontiers and future},
    author={Chu, Zheng and Chen, Jingchang and Chen, Qianglong and Yu, Weijiang and He, Tao and Wang, Haotian and Peng, Weihua and Liu, Ming and Qin, Bing and Liu, Ting},
    journal={arXiv preprint arXiv:2309.15402},
    year={2023}
}

@article{yu-etal:2023towards,
    title={Towards better chain-of-thought prompting strategies: A survey},
    author={Yu, Zihan and He, Liang and Wu, Zhen and Dai, Xinyu and Chen, Jiajun},
    journal={arXiv preprint arXiv:2310.04959},
    year={2023}
}

@article{zhang-etal:2023igniting,
    title={Igniting Language Intelligence: The Hitchhiker's Guide From Chain-of-Thought Reasoning to Language Agents},
    author={Zhuosheng Zhang and Yao Yao and Aston Zhang and Xiangru Tang and Xinbei Ma and Zhiwei He and Yiming Wang and Mark Gerstein and Rui Wang and Gongshen Liu and Hai Zhao},
    journal={arXiv preprint arXiv:2311.11797},
    year={2023}
}

@inproceedings{zhou-etal:least,
    title={Least-to-Most Prompting Enables Complex Reasoning in Large Language Models},
    author={Denny Zhou and Nathanael Sch{\"{a}}rli and Le Hou and Jason Wei and Nathan Scales and Xuezhi Wang and Dale Schuurmans and Claire Cui and Olivier Bousquet and Quoc V. Le and Ed H. Chi},
    booktitle={Proceedings of The Eleventh International Conference on Learning Representations},
    year={2023}
}

@book{frensch-and-funke:2014complex,
    title={Complex problem solving: The European perspective},
    author={Frensch, Peter A and Funke, Joachim},
    year={2014},
    publisher={Psychology Press}
}

@inproceedings{creswell-etal:2023selection,
    title={Selection-Inference: Exploiting Large Language Models for Interpretable Logical Reasoning},
    author={Antonia Creswell and Murray Shanahan and Irina Higgins},
    booktitle={Proceedings of  The Eleventh International Conference on Learning Representations},
    year={2023}
}

@inproceedings{dua-etal:2022successive,
    title={Successive Prompting for Decomposing Complex Questions},
    author={Dua, Dheeru and Gupta, Shivanshu and Singh, Sameer and Gardner, Matt},
    booktitle={Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing},
    pages={1251--1265},
    year={2022}
}

@article{mavi-etal:2024multi,
    title={Multi-hop Question Answering},
    author={Vaibhav Mavi and Anubhav Jangra and Adam Jatowt},
    journal={Foundations and Trends{\textregistered} in Information Retrieval},
    volume={17},
    number={5},
    pages={457--586},
    year={2024},
    publisher={Now Publishers, Inc.}
}

@inproceedings{khot2023:decomposed,
    title={Decomposed Prompting: A Modular Approach for Solving Complex Tasks},
    author={Khot, Tushar and Trivedi, Harsh and Finlayson, Matthew and Fu, Yao and Richardson, Kyle and Clark, Peter and Sabharwal, Ashish},
    booktitle={Proceedings of The Eleventh International Conference on Learning Representations},
    year={2023}
}

@inproceedings{andreas-etal:2016neural,
    title={Neural module networks},
    author={Andreas, Jacob and Rohrbach, Marcus and Darrell, Trevor and Klein, Dan},
    booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
    pages={39--48},
    year={2016}
}

@article{talmor-etal:2018web,
    title={The web as a knowledge-base for answering complex questions},
    author={Talmor, Alon and Berant, Jonathan},
    journal={arXiv preprint arXiv:1803.06643},
    year={2018}
}

@inproceedings{min-etal:2019multi,
    title={Multi-hop Reading Comprehension through Question Decomposition and Rescoring},
    author={Min, Sewon and Zhong, Victor and Zettlemoyer, Luke and Hajishirzi, Hannaneh},
    booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
    pages={6097--6109},
    year={2019}
}

@inproceedings{lake-etal:2018generalization,
    title={Generalization without systematicity: On the compositional skills of sequence-to-sequence recurrent networks},
    author={Lake, Brenden and Baroni, Marco},
    booktitle={International conference on machine learning},
    pages={2873--2882},
    year={2018},
    organization={PMLR}
}

@inproceedings{drozdov-etal:2022compositional,
    title={Compositional semantic parsing with large language models},
    author={Drozdov, Andrew and Sch{\"a}rli, Nathanael and Aky{\"u}rek, Ekin and Scales, Nathan and Song, Xinying and Chen, Xinyun and Bousquet, Olivier and Zhou, Denny},
    booktitle={Proceedings of The Eleventh International Conference on Learning Representations},
    year={2022}
}

@inproceedings{press-etal:2023measuring,
    title={Measuring and Narrowing the Compositionality Gap in Language Models},
    author={Press, Ofir and Zhang, Muru and Min, Sewon and Schmidt, Ludwig and Smith, Noah A and Lewis, Mike},
    booktitle={Findings of the Association for Computational Linguistics: EMNLP 2023},
    pages={5687--5711},
    year={2023}
}

@inproceedings{brill:1992simple,
    title={A simple rule-based part of speech tagger},
    author={Brill, Eric},
    booktitle={Speech and Natural Language: Proceedings of a Workshop Held at Harriman, New York, February 23-26, 1992},
    year={1992}
}

@article{madaan-etal:2024self,
    title={Self-refine: Iterative refinement with self-feedback},
    author={Aman Madaan and Niket Tandon and Prakhar Gupta and Skyler Hallinan and Luyu Gao and Sarah Wiegreffe and Uri Alon and Nouha Dziri and Shrimai Prabhumoye and Yiming Yang and Shashank Gupta and Bodhisattwa Prasad Majumder and Katherine Hermann and Sean Welleck and Amir Yazdanbakhsh and Peter Clark},
    journal={Advances in Neural Information Processing Systems},
    volume={36},
    year={2024}
}

@inproceedings{welleck-etal:2023generating,
    title={Generating Sequences by Learning to Self-Correct},
    author={Welleck, Sean and Lu, Ximing and West, Peter and Brahman, Faeze and Shen, Tianxiao and Khashabi, Daniel and Choi, Yejin},
    booktitle={Proceedings of The Eleventh International Conference on Learning Representations},
    year={2023}
}

@inproceedings{schick-etal:2023peer,
    title={{PEER}: A Collaborative Language Model},
    author={Timo Schick and Jane A. Yu and Zhengbao Jiang and Fabio Petroni and Patrick Lewis and Gautier Izacard and Qingfei You and Christoforos Nalmpantis and Edouard Grave and Sebastian Riedel},
    booktitle={Proceedings of The Eleventh International Conference on Learning Representations},
    year={2023}
}

@article{ganguli-etal:2023capacity,
    title={The capacity for moral self-correction in large language models},
    author={Deep Ganguli and Amanda Askell and Nicholas Schiefer and Thomas I. Liao and Kamile Lukosiute and Anna Chen and Anna Goldie and Azalia Mirhoseini and Catherine Olsson and Danny Hernandez and Dawn Drain and Dustin Li and Eli Tran{-}Johnson and Ethan Perez and Jackson Kernion and Jamie Kerr and Jared Mueller and Joshua Landau and Kamal Ndousse and Karina Nguyen and Liane Lovitt and Michael Sellitto and Nelson Elhage and Noem{\'{\i}} Mercado and Nova DasSarma and Oliver Rausch and Robert Lasenby and Robin Larson and Sam Ringer and Sandipan Kundu and Saurav Kadavath and Scott Johnston and Shauna Kravec and Sheer El Showk and Tamera Lanham and Timothy Telleen{-}Lawton and Tom Henighan and Tristan Hume and Yuntao Bai and Zac Hatfield{-}Dodds and Ben Mann and Dario Amodei and Nicholas Joseph and Sam McCandlish and Tom Brown and Christopher Olah and Jack Clark and Samuel R. Bowman and Jared Kaplan},
    journal={arXiv preprint arXiv:2302.07459},
    year={2023}
}

@article{li2023-etal:deliberate,
    title={Deliberate then generate: Enhanced prompting framework for text generation},
    author={Li, Bei and Wang, Rui and Guo, Junliang and Song, Kaitao and Tan, Xu and Hassan, Hany and Menezes, Arul and Xiao, Tong and Bian, Jiang and Zhu, JingBo},
    journal={arXiv preprint arXiv:2305.19835},
    year={2023}
}

@article{marcus-etal:1993negative,
    title={Negative evidence in language acquisition},
    author={Marcus, Gary F},
    journal={Cognition},
    volume={46},
    number={1},
    pages={53--85},
    year={1993},
    publisher={Elsevier}
}

@article{wang-etal:2022rationale,
    title={Rationale-augmented ensembles in language models},
    author={Wang, Xuezhi and Wei, Jason and Schuurmans, Dale and Le, Quoc and Chi, Ed and Zhou, Denny},
    journal={arXiv preprint arXiv:2207.00747},
    year={2022}
}

@inproceedings{wang-etal:2023self,
    title={Self-Consistency Improves Chain of Thought Reasoning in Language Models},
    author={Wang, Xuezhi and Wei, Jason and Schuurmans, Dale and Le, Quoc V and Chi, Ed H and Narang, Sharan and Chowdhery, Aakanksha and Zhou, Denny},
    booktitle={Proceedings of The Eleventh International Conference on Learning Representations},
    year={2023}
}

@article{pan-etal:2024automatically,
    title = "Automatically Correcting Large Language Models: Surveying the Landscape of Diverse Automated Correction Strategies",
    author = "Pan, Liangming  and Saxon, Michael  and Xu, Wenda  and Nathani, Deepak  and Wang, Xinyi  and Wang, William Yang",
    journal = "Transactions of the Association for Computational Linguistics",
    volume = "12",
    year = "2024",
    publisher = "MIT Press",
    pages = "484--506"
}

@article{fernandes-etal:2023bridging,
    title={Bridging the gap: A survey on integrating (human) feedback for natural language generation},
    author={Patrick Fernandes and Aman Madaan and Emmy Liu and Ant{\'{o}}nio Farinhas and Pedro Henrique Martins and Amanda Bertsch and Jos{\'{e}} G. C. de Souza and Shuyan Zhou and Tongshuang Wu and Graham Neubig and Andr{\'{e}} F. T. Martins},
    journal={Transactions of the Association for Computational Linguistics},
    volume={11},
    pages={1643--1668},
    year={2023},
    publisher={MIT Press}
}

@article{uesato-etal:2022solving,
    title={Solving math word problems with process-and outcome-based feedback},
    author={Uesato, Jonathan and Kushman, Nate and Kumar, Ramana and Song, Francis and Siegel, Noah and Wang, Lisa and Creswell, Antonia and Irving, Geoffrey and Higgins, Irina},
    journal={arXiv preprint arXiv:2211.14275},
    year={2022}
}

@inproceedings{lightman-etal:2024lets,
    title={Let's Verify Step by Step},
    author={Hunter Lightman and Vineet Kosaraju and Yuri Burda and Harrison Edwards and Bowen Baker and Teddy Lee and Jan Leike and John Schulman and Ilya Sutskever and Karl Cobbe},
    booktitle={The Twelfth International Conference on Learning Representations},
    year={2024}
}

@article{li-etal:2022survey,
    title={A survey on retrieval-augmented text generation},
    author={Li, Huayang and Su, Yixuan and Cai, Deng and Wang, Yan and Liu, Lemao},
    journal={arXiv preprint arXiv:2202.01110},
    year={2022}
}

@article{gao-etal:2023retrieval,
    title={Retrieval-augmented generation for large language models: A survey},
    author={Gao, Yunfan and Xiong, Yun and Gao, Xinyu and Jia, Kangxiang and Pan, Jinliu and Bi, Yuxi and Dai, Yi and Sun, Jiawei and Wang, Haofen},
    journal={arXiv preprint arXiv:2312.10997},
    year={2023}
}

@inproceedings{franklin-and-graesser:1996agent,
    title={Is it an Agent, or just a Program?: A Taxonomy for Autonomous Agents},
  author={Franklin, Stan and Graesser, Art},
    booktitle={International workshop on agent theories, architectures, and languages},
    pages={21--35},
    year={1996},
    organization={Springer}
}

@article{parisi-etal:2022talm,
    title={Talm: Tool augmented language models},
    author={Parisi, Aaron and Zhao, Yao and Fiedel, Noah},
    journal={arXiv preprint arXiv:2205.12255},
    year={2022}
}

@inproceedings{gao-etal:2023pal,
    title={Pal: Program-aided language models},
    author={Gao, Luyu and Madaan, Aman and Zhou, Shuyan and Alon, Uri and Liu, Pengfei and Yang, Yiming and Callan, Jamie and Neubig, Graham},
    booktitle={International Conference on Machine Learning},
    pages={10764--10799},
    year={2023},
    organization={PMLR}
}

@article{schick-etal:2024toolformer,
    title={Toolformer: Language models can teach themselves to use tools},
    author={Schick, Timo and Dwivedi-Yu, Jane and Dess{\`\i}, Roberto and Raileanu, Roberta and Lomeli, Maria and Hambro, Eric and Zettlemoyer, Luke and Cancedda, Nicola and Scialom, Thomas},
    journal={Advances in Neural Information Processing Systems},
    volume={36},
    year={2024}
}

@article{elsken-etal:2019nas,
    author  = {Thomas Elsken and Jan Hendrik Metzen and Frank Hutter},
    title   = {Neural Architecture Search: A Survey},
    journal = {Journal of Machine Learning Research},
    year    = {2019},
    volume  = {20},
    number  = {55},
    pages   = {1--21}
}

@inproceedings{zhou-etal:2023large,
    title = {Large Language Models are Human-Level Prompt Engineers},
    author = {Yongchao Zhou and Andrei Ioan Muresanu and Ziwen Han and Keiran Paster and Silviu Pitis and Harris Chan and Jimmy Ba},
    booktitle = {The Eleventh International Conference on Learning Representations },
    year = {2023}
}

@article{jiang-etal:2020can,
    title={How can we know what language models know?},
    author={Jiang, Zhengbao and Xu, Frank F and Araki, Jun and Neubig, Graham},
    journal={Transactions of the Association for Computational Linguistics},
    volume={8},
    pages={423--438},
    year={2020},
    publisher={MIT Press}
}

@inproceedings{prasad-etal:2023grips,
    title={GrIPS: Gradient-free, Edit-based Instruction Search for Prompting Large Language Models},
    author={Prasad, Archiki and Hase, Peter and Zhou, Xiang and Bansal, Mohit},
    booktitle={Proceedings of the 17th Conference of the European Chapter of the Association for Computational Linguistics},
    pages={3845--3864},
    year={2023}
}

@inproceedings{pryzant-etal:2023automatic,
    title={Automatic Prompt Optimization with "Gradient Descent" and Beam Search},
    author={Pryzant, Reid and Iter, Dan and Li, Jerry and Lee, Yin Tat and Zhu, Chenguang and Zeng, Michael},
    booktitle={The 2023 Conference on Empirical Methods in Natural Language Processing},
    year={2023}
}

@inproceedings{guo-etal:2024connecting,
    title={Connecting Large Language Models with Evolutionary Algorithms Yields Powerful Prompt Optimizers},
    author={Qingyan Guo and Rui Wang and Junliang Guo and Bei Li and Kaitao Song and Xu Tan and Guoqing Liu and Jiang Bian and Yujiu Yang},
    booktitle={The Twelfth International Conference on Learning Representations},
    year={2024}
}

@inproceedings{deng-etal:2022rlprompt,
    title={RLPrompt: Optimizing Discrete Text Prompts with Reinforcement Learning},
    author={Deng, Mingkai and Wang, Jianyu and Hsieh, Cheng-Ping and Wang, Yihan and Guo, Han and Shu, Tianmin and Song, Meng and Xing, Eric and Hu, Zhiting},
    booktitle={Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing},
    pages={3369--3391},
    year={2022}
}

@inproceedings{liu-etal:2022makes,
    title={What Makes Good In-Context Examples for GPT-3?},
    author={Liu, Jiachang and Shen, Dinghan and Zhang, Yizhe and Dolan, William B and Carin, Lawrence and Chen, Weizhu},
    booktitle={Proceedings of Deep Learning Inside Out (DeeLIO 2022): The 3rd Workshop on Knowledge Extraction and Integration for Deep Learning Architectures},
    pages={100--114},
    year={2022}
}

@inproceedings{rubin-etal:2022learning,
    title={Learning To Retrieve Prompts for In-Context Learning},
    author={Rubin, Ohad and Herzig, Jonathan and Berant, Jonathan},
    booktitle={Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
    pages={2655--2671},
    year={2022}
}

@inproceedings{zhang-etal:2023automatic,
    title={Automatic Chain of Thought Prompting in Large Language Models},
    author={Zhuosheng Zhang and Aston Zhang and Mu Li and Alex Smola},
    booktitle={The Eleventh International Conference on Learning Representations },
    year={2023}
}

@article{snell-etal:2022learning,
    title={Learning by distilling context},
    author={Snell, Charlie and Klein, Dan and Zhong, Ruiqi},
    journal={arXiv preprint arXiv:2209.15189},
    year={2022}
}

@article{askell-etal:2021general,
    title={A general language assistant as a laboratory for alignment},
    author={Amanda Askell and Yuntao Bai and Anna Chen and Dawn Drain and Deep Ganguli and Tom Henighan and Andy Jones and Nicholas Joseph and Benjamin Mann and Nova DasSarma and Nelson Elhage and Zac Hatfield{-}Dodds and Danny Hernandez and Jackson Kernion and Kamal Ndousse and Catherine Olsson and Dario Amodei and Tom B. Brown and Jack Clark and Sam McCandlish and Chris Olah and Jared Kaplan},
    journal={arXiv preprint arXiv:2112.00861},
    year={2021}
}

@inproceedings{li-liang:2021prefix,
    title={Prefix-Tuning: Optimizing Continuous Prompts for Generation},
    author={Li, Xiang Lisa and Liang, Percy},
    booktitle={Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},
    pages={4582--4597},
    year={2021}
}

@inproceedings{lester-etal:2021power,
    title={The Power of Scale for Parameter-Efficient Prompt Tuning},
    author={Lester, Brian and Al-Rfou, Rami and Constant, Noah},
    booktitle={Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},
    pages={3045--3059},
    year={2021}
}

@article{liu-etal:2023gpt,
    title={GPT understands, too},
    author={Liu, Xiao and Zheng, Yanan and Du, Zhengxiao and Ding, Ming and Qian, Yujie and Yang, Zhilin and Tang, Jie},
    journal={AI Open},
    year={2023},
    publisher={Elsevier}
}

@article{mu-etal:2024learning,
    title={Learning to compress prompts with gist tokens},
    author={Mu, Jesse and Li, Xiang and Goodman, Noah},
    journal={Advances in Neural Information Processing Systems},
    volume={36},
    year={2024}
}

@article{lialin-etal:2023scaling,
    title={Scaling down to scale up: A guide to parameter-efficient fine-tuning},
    author={Lialin, Vladislav and Deshpande, Vijeta and Rumshisky, Anna},
    journal={arXiv preprint arXiv:2303.15647},
    year={2023}
}

@inproceedings{wingate-etal:2022prompt,
    title={Prompt Compression and Contrastive Conditioning for Controllability and Toxicity Reduction in Language Models},
    author={Wingate, David and Shoeybi, Mohammad and Sorensen, Taylor},
    booktitle={Findings of the Association for Computational Linguistics: EMNLP 2022},
    pages={5621--5634},
    year={2022}
}

@inproceedings{chevalier-etal:2023adapting,
    title={Adapting Language Models to Compress Contexts},
    author={Chevalier, Alexis and Wettig, Alexander and Ajith, Anirudh and Chen, Danqi},
    booktitle={Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},
    pages={3829--3846},
    year={2023}
}

@inproceedings{li-etal:2023compressing,
    title={Compressing Context to Enhance Inference Efficiency of Large Language Models},
    author={Li, Yucheng and Dong, Bo and Guerin, Frank and Lin, Chenghua},
    booktitle={Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},
    pages={6342--6353},
    year={2023}
}

@inproceedings{jiang-etal:2023llmlingua,
    title={LLMLingua: Compressing Prompts for Accelerated Inference of Large Language Models},
    author={Jiang, Huiqiang and Wu, Qianhui and Lin, Chin-Yew and Yang, Yuqing and Qiu, Lili},
    booktitle={Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},
    pages={13358--13376},
    year={2023}
}

@inproceedings{li:2023practical,
    title={A Practical Survey on Zero-Shot Prompt Design for In-Context Learning},
    author={Li, Yinheng},
    booktitle={Proceedings of the 14th International Conference on Recent Advances in Natural Language Processing},
    pages={641--647},
    year={2023}
}

@article{dong-etal:2022survey,
    title={A survey on in-context learning},
    author={Dong, Qingxiu and Li, Lei and Dai, Damai and Zheng, Ce and Wu, Zhiyong and Chang, Baobao and Sun, Xu and Xu, Jingjing and Sui, Zhifang},
    journal={arXiv preprint arXiv:2301.00234},
    year={2022}
}

@article{chang-etal:2024efficient,
    title={Efficient Prompting Methods for Large Language Models: A Survey},
    author={Chang, Kaiyan and Xu, Songcheng and Wang, Chenglong and Luo, Yingfeng and Xiao, Tong and Zhu, Jingbo},
    journal={arXiv preprint arXiv:2404.01077},
    year={2024}
}

@article{liu2023-etal:prompting,
    title={Prompting frameworks for large language models: A survey},
    author={Liu, Xiaoxia and Wang, Jingyi and Sun, Jun and Yuan, Xiaohan and Dong, Guoliang and Di, Peng and Wang, Wenhai and Wang, Dongxia},
    journal={arXiv preprint arXiv:2311.12785},
    year={2023}
}

@article{chen-etal:2023unleashing,
    title={Unleashing the potential of prompt engineering in Large Language Models: a comprehensive review},
    author={Chen, Banghao and Zhang, Zhaofeng and Langren{\'e}, Nicolas and Zhu, Shengxin},
    journal={arXiv preprint arXiv:2310.14735},
    year={2023}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CHAPTER 11

@inproceedings{hu-etal:2022lora,
    title={Lo{RA}: Low-Rank Adaptation of Large Language Models},
    author={Edward J Hu and yelong shen and Phillip Wallis and Zeyuan Allen-Zhu and Yuanzhi Li and Shean Wang and Lu Wang and Weizhu Chen},
    booktitle={International Conference on Learning Representations},
    year={2022}
}

@InProceedings{houlsby-etla:2019parameter,
  title = 	 {Parameter-Efficient Transfer Learning for {NLP}},
  author =   {Houlsby, Neil and Giurgiu, Andrei and Jastrzebski, Stanislaw and Morrone, Bruna and De Laroussilhe, Quentin and Gesmundo, Andrea and Attariyan, Mona and Gelly, Sylvain},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {2790--2799},
  year = 	 {2019},
  publisher =    {PMLR},
}

@article{han-etal:2024parameter,
    title={Parameter-efficient fine-tuning for large models: A comprehensive survey},
    author={Zeyu Han and Chao Gao and Jinyang Liu and Jeff Zhang and Sai Qian Zhang},
    journal={arXiv preprint arXiv:2403.14608},
    year={2024}
}

@inproceedings{bach-etal:2022promptsource,
    title={PromptSource: An Integrated Development Environment and Repository for Natural Language Prompts},
    author={Stephen H. Bach and
                  Victor Sanh and
                  Zheng Xin Yong and
                  Albert Webson and
                  Colin Raffel and
                  Nihal V. Nayak and
                  Abheesht Sharma and
                  Taewoon Kim and
                  M. Saiful Bari and
                  Thibault F{\'{e}}vry and
                  Zaid Alyafeai and
                  Manan Dey and
                  Andrea Santilli and
                  Zhiqing Sun and
                  Srulik Ben{-}David and
                  Canwen Xu and
                  Gunjan Chhablani and
                  Han Wang and
                  Jason Alan Fries and
                  Maged Saeed AlShaibani and
                  Shanya Sharma and
                  Urmish Thakker and
                  Khalid Almubarak and
                  Xiangru Tang and
                  Dragomir R. Radev and
                  Mike Tian{-}Jian Jiang and
                  Alexander M. Rush},
    booktitle={Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics: System Demonstrations},
    pages={93--104},
    year={2022}
}

@inproceedings{wang-etal:2022super,
    title={Super-NaturalInstructions: Generalization via Declarative Instructions on 1600+ NLP Tasks},
    author={Yizhong Wang and
                  Swaroop Mishra and
                  Pegah Alipoormolabashi and
                  Yeganeh Kordi and
                  Amirreza Mirzaei and
                  Atharva Naik and
                  Arjun Ashok and
                  Arut Selvan Dhanasekaran and
                  Anjana Arunkumar and
                  David Stap and
                  Eshaan Pathak and
                  Giannis Karamanolakis and
                  Haizhi Gary Lai and
                  Ishan Purohit and
                  Ishani Mondal and
                  Jacob Anderson and
                  Kirby Kuznia and
                  Krima Doshi and
                  Kuntal Kumar Pal and
                  Maitreya Patel and
                  Mehrad Moradshahi and
                  Mihir Parmar and
                  Mirali Purohit and
                  Neeraj Varshney and
                  Phani Rohitha Kaza and
                  Pulkit Verma and
                  Ravsehaj Singh Puri and
                  Rushang Karia and
                  Savan Doshi and
                  Shailaja Keyur Sampat and
                  Siddhartha Mishra and
                  Sujan Reddy A and
                  Sumanta Patro and
                  Tanay Dixit and
                  Xudong Shen},
    booktitle={Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing},
    pages={5085--5109},
    year={2022}
}

@inproceedings{mishra-etal:2022cross,
    title={Cross-Task Generalization via Natural Language Crowdsourcing Instructions},
    author={Mishra, Swaroop and Khashabi, Daniel and Baral, Chitta and Hajishirzi, Hannaneh},
    booktitle={Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={3470--3487},
    year={2022}
}

@inproceedings{joshi-etal:2017triviaqa,
    title={TriviaQA: A Large Scale Distantly Supervised Challenge Dataset for Reading Comprehension},
    author={Joshi, Mandar and Choi, Eunsol and Weld, Daniel S and Zettlemoyer, Luke},
    booktitle={Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={1601--1611},
    year={2017}
}

@inproceedings{wang-etal:2023selfinstruct,
    title={Self-Instruct: Aligning Language Models with Self-Generated Instructions},
    author={Wang, Yizhong and Kordi, Yeganeh and Mishra, Swaroop and Liu, Alisa and Smith, Noah A and Khashabi, Daniel and Hajishirzi, Hannaneh},
    booktitle={Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={13484--13508},
    year={2023}
}

@inproceedings{honovich-etal:2023unnatural,
    title={Unnatural Instructions: Tuning Language Models with (Almost) No Human Labor},
    author={Honovich, Or and Scialom, Thomas and Levy, Omer and Schick, Timo},
    booktitle={Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={14409--14428},
    year={2023}
}

@inproceedings{longpre-etal:2023flan,
    title={The flan collection: Designing data and methods for effective instruction tuning},
    author={Shayne Longpre and Le Hou and Tu Vu and Albert Webson and Hyung Won Chung and Yi Tay and Denny Zhou and Quoc V. Le and Barret Zoph and Jason Wei and Adam Roberts},
    booktitle={International Conference on Machine Learning},
    pages={22631--22648},
    year={2023},
    organization={PMLR}
}

@misc{taori-etal:2023alpaca,
    author = {Rohan Taori and Ishaan Gulrajani and Tianyi Zhang and Yann Dubois and Xuechen Li and Carlos Guestrin and Percy Liang and Tatsunori B. Hashimoto },
    title = {Stanford Alpaca: An Instruction-following LLaMA model},
    year = {2023},
    publisher = {GitHub},
    journal = {GitHub repository},
    howpublished = {\url{https://github.com/tatsu-lab/stanford_alpaca}}
}

@misc{chiang-etal:2023vicuna,
    title = {Vicuna: An Open-Source Chatbot Impressing GPT-4 with 90\%* ChatGPT Quality},
    url = {https://lmsys.org/blog/2023-03-30-vicuna/},
    author = {Chiang, Wei-Lin and Li, Zhuohan and Lin, Zi and Sheng, Ying and Wu, Zhanghao and Zhang, Hao and Zheng, Lianmin and Zhuang, Siyuan and Zhuang, Yonghao and Gonzalez, Joseph E. and Stoica, Ion and Xing, Eric P.},
    month = {March},
    year = {2023}
}

@inproceedings{xu-etal:2024wizardlm,
    title={WizardLM: Empowering large pre-trained language models to follow complex instructions},
    author={Xu, Can and Sun, Qingfeng and Zheng, Kai and Geng, Xiubo and Zhao, Pu and Feng, Jiazhan and Tao, Chongyang and Lin, Qingwei and Jiang, Daxin},
    booktitle={The Twelfth International Conference on Learning Representations},
    year={2024}
}

@article{chen-etal:2024self,
    title={Self-play fine-tuning converts weak language models to strong language models},
    author={Chen, Zixiang and Deng, Yihe and Yuan, Huizhuo and Ji, Kaixuan and Gu, Quanquan},
    journal={arXiv preprint arXiv:2401.01335},
    year={2024}
}

@article{gunasekar-etal:2023textbooks,
    title={Textbooks are all you need},
    author={Suriya Gunasekar and Yi Zhang and Jyoti Aneja and Caio C{\'{e}}sar Teodoro Mendes and Allie Del Giorno and Sivakanth Gopi and Mojan Javaheripi and Piero Kauffmann and Gustavo de Rosa and Olli Saarikivi and Adil Salim and Shital Shah and Harkirat Singh Behl and Xin Wang and S{\'{e}}bastien Bubeck and Ronen Eldan and Adam Tauman Kalai and Yin Tat Lee and  Yuanzhi Li},
    journal={arXiv preprint arXiv:2306.11644},
    year={2023}
}

@misc{allal-etal:2024cosmopedia,
    author = {Loubna Ben Allal and Anton Lozhkov  and Daniel van Strien},
    title = {cosmopedia: how to create large-scale synthetic data for pre-training},
    year = {2024},
    journal = {Hugging Face},
    howpublished = {\url{https://huggingface.co/blog/cosmopedia}}
}

@article{charniak:1997statistical,
    title={Statistical parsing with a context-free grammar and word statistics},
    author={Charniak, Eugene},
    journal={AAAI/IAAI},
    volume={2005},
    number={598-603},
    pages={18},
    year={1997}
}

@inproceedings{chen-etal:2024alpagasus,
    title={AlpaGasus: Training a Better Alpaca with Fewer Data},
    author={Lichang Chen and Shiyang Li and Jun Yan and Hai Wang and Kalpa Gunaratna and Vikas Yadav and Zheng Tang and Vijay Srinivasan and Tianyi Zhou and Heng Huang and Hongxia Jin},
    booktitle={The Twelfth International Conference on Learning Representations},
    year={2024}
}

@article{zhao-etal:2024long,
    title={Long is more for alignment: A simple but tough-to-beat baseline for instruction fine-tuning},
    author={Zhao, Hao and Andriushchenko, Maksym and Croce, Francesco and Flammarion, Nicolas},
    journal={arXiv preprint arXiv:2402.04833},
    year={2024}
}

@article{ge-etal:2024clustering,
    title={Clustering and ranking: Diversity-preserved instruction selection through expert-aligned quality estimation},
    author={Ge, Yuan and Liu, Yilun and Hu, Chi and Meng, Weibin and Tao, Shimin and Zhao, Xiaofeng and Ma, Hongxia and Zhang, Li and Chen, Boxing and Yang, Hao and Li,Bei and Xiao, Tong and Zhu, Jingbo},
    journal={arXiv preprint arXiv:2402.18191},
    year={2024}
}

@article{xia-etal:2024less,
    title={Less: Selecting influential data for targeted instruction tuning},
    author={Xia, Mengzhou and Malladi, Sadhika and Gururangan, Suchin and Arora, Sanjeev and Chen, Danqi},
    journal={arXiv preprint arXiv:2402.04333},
    year={2024}
}

@article{hewitt-etal:2024instruction,
    title={Instruction following without instruction tuning},
    author={Hewitt, John and Liu, Nelson F and Liang, Percy and Manning, Christopher D},
    journal={arXiv preprint arXiv:2409.14254},
    year={2024}
}

@inproceedings{liu-etal:2024statistical,
   title={Statistical Rejection Sampling Improves Preference Optimization},
   author={Tianqi Liu and Yao Zhao and Rishabh Joshi and Misha Khalman and Mohammad Saleh and Peter J Liu and Jialu Liu},
   booktitle={The Twelfth International Conference on Learning Representations},
   year={2024}
}

@inproceedings{wang-etal:2024esrl,
    title={Esrl: Efficient sampling-based reinforcement learning for sequence generation},
    author={Wang, Chenglong and Zhou, Hang and Hu, Yimin and Huo, Yifu and Li, Bei and Liu, Tongran and Xiao, Tong and Zhu, Jingbo},
    booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
    pages={19107--19115},
    year={2024}
}

@article{wang-etal:2023far,
    title={How far can camels go? exploring the state of instruction tuning on open resources},
    author={Yizhong Wang and Hamish Ivison and Pradeep Dasigi and Jack Hessel and Tushar Khot and Khyathi Raghavi Chandu and David Wadden and Kelsey MacMillan and Noah A. Smith and Iz Beltagy and Hannaneh Hajishirzi},
    journal={Advances in Neural Information Processing Systems},
    volume={36},
    pages={74764--74786},
    year={2023}
}

@misc{teknium:2023openhermes,
    title = {OpenHermes 2.5: An Open Dataset of Synthetic Data for Generalist LLM Assistants},
    author = {Teknium},
    year = {2023},
    publisher = {HuggingFace},
    url = {https://huggingface.co/datasets/teknium/OpenHermes-2.5}
}

@inproceedings{hendrycks-etal:2020pretrained,
    title={Pretrained Transformers Improve Out-of-Distribution Robustness},
    author={Hendrycks, Dan and Liu, Xiaoyuan and Wallace, Eric and Dziedzic, Adam and Krishnan, Rishabh and Song, Dawn},
    booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
    pages={2744--2751},
    year={2020}
}

@article{kung-and-peng:2023models,
    title={Do models really learn to follow instructions? an empirical study of instruction tuning},
    author={Kung, Po-Nien and Peng, Nanyun},
    journal={arXiv preprint arXiv:2305.11383},
    year={2023}
}

@inproceedings{radford-etal:2021learning,
    title={Learning transferable visual models from natural language supervision},
    author={Alec Radford and Jong Wook Kim and Chris Hallacy and Aditya Ramesh and Gabriel Goh and Sandhini Agarwal and Girish Sastry and Amanda Askell and Pamela Mishkin and Jack Clark and Gretchen Krueger and Ilya Sutskever},
    booktitle={International conference on machine learning},
    pages={8748--8763},
    year={2021},
    organization={PMLR}
}

@misc{hewitt:2024instructionblog,
    title = {Instruction Following without Instruction Tuning},
    author = {John Hewitt},
    year = {2024},
    publisher = {Stanford},
    url = {https://nlp.stanford.edu/~johnhew/instruction-following.html}
}

@misc{burns-etal:2023weakblog,
    title = {Weak-to-strong generalization},
    author = {Collin Burns and Jan Leike and Leopold Aschenbrenner and Jeffrey Wu and Pavel Izmailov and Leo Gao and Bowen Baker and Jan Hendrik Kirchner},
    year = {2023},
    publisher = {OpenAI},
    url = {https://https://openai.com/index/weak-to-strong-generalization}
}

@article{burns-etal:2023weak,
    title={Weak-to-strong generalization: Eliciting strong capabilities with weak supervision},
    author={Burns, Collin and Izmailov, Pavel and Kirchner, Jan Hendrik and Baker, Bowen and Gao, Leo and Aschenbrenner, Leopold and Chen, Yining and Ecoffet, Adrien and Joglekar, Manas and Leike, Jan and Sutskever, Ilya and Wu, Jeff},
    journal={arXiv preprint arXiv:2312.09390},
    year={2023}
}

@misc{aschenbrenner:2024situational,
    title = {SITUATIONAL AWARENESS: The Decade Ahead},
    author = {Leopold Aschenbrenner},
    year = {2024},
    url = {https://situational-awareness.ai/}
}

@article{szepesvari:2010algorithms,
    title={Algorithms for Reinforcement Learning},
    author={Szepesv{\'a}ri, Csaba},
    journal={Synthesis Lectures on Artificial Intelligence and Machine Learning},
    volume={4},
    number={1},
    pages={1--103},
    year={2010},
    publisher={Springer Science and Business Media LLC}
}

@article{bradley-and-terry:rank,
    author = {Ralph Allan Bradley and Milton E. Terry},
    journal = {Biometrika},
    number = {3/4},
    pages = {324--345},
    title = {Rank Analysis of Incomplete Block Designs: I. The Method of Paired Comparisons},
    volume = {39},
    year = {1952}
}

@inproceedings{schulman-etal:2015trust,
    title={Trust region policy optimization},
    author={Schulman, John and Levine, Sergey and Moritz, Philipp and Jordan, Michael and Abbeel, Pieter},
    booktitle={Proceedings of the 32nd International Conference on International Conference on Machine Learning-Volume 37},
    pages={1889--1897},
    year={2015}
}

@article{schulman-etal:2017proximal,
    title={Proximal policy optimization algorithms},
    author={Schulman, John and Wolski, Filip and Dhariwal, Prafulla and Radford, Alec and Klimov, Oleg},
    journal={arXiv preprint arXiv:1707.06347},
    year={2017}
}

@inproceedings{burges-etal:2005learning,
    title={Learning to rank using gradient descent},
    author={Burges, Chris and Shaked, Tal and Renshaw, Erin and Lazier, Ari and Deeds, Matt and Hamilton, Nicole and Hullender, Greg},
    booktitle={Proceedings of the 22nd international conference on Machine learning},
    pages={89--96},
    year={2005}
}

@inproceedings{cao-etal:2007learning,
    title={Learning to rank: from pairwise approach to listwise approach},
    author={Cao, Zhe and Qin, Tao and Liu, Tie-Yan and Tsai, Ming-Feng and Li, Hang},
    booktitle={Proceedings of the 24th international conference on Machine learning},
    pages={129--136},
    year={2007}
}

@article{plackett:1975analysis,
    title={The analysis of permutations},
    author={Plackett, Robin L},
    journal={Journal of the Royal Statistical Society Series C: Applied Statistics},
    volume={24},
    number={2},
    pages={193--202},
    year={1975},
    publisher={Oxford University Press}
}

@book{li2011:learning,
    title={Learning to Rank for Information Retrieval and Natural Language Processing},
    author={Li, Hang},
    isbn={9781608457076},
    series={Online access: Morgan \& Claypool Synthesis Collection Five},
    year={2011},
    publisher={Morgan \& Claypool Publishers}
}

@article{liu:2009learning,
    title={Learning to rank for information retrieval},
    author={Liu, Tie-Yan},
    journal={Foundations and Trends{\textregistered} in Information Retrieval},
    volume={3},
    number={3},
    pages={225--331},
    year={2009},
    publisher={Now Publishers, Inc.}
}

@article{eisenstein-etal:2023helping,
    title={Helping or herding? reward model ensembles mitigate but do not eliminate reward hacking},
    author={Eisenstein, Jacob and Nagpal, Chirag and Agarwal, Alekh and Beirami, Ahmad and D'Amour, Alex and Dvijotham, DJ and Fisch, Adam and Heller, Katherine and Pfohl, Stephen and Ramachandran, Deepak and Shaw, Peter},
    journal={arXiv preprint arXiv:2312.09244},
    year={2023}
}

@inproceedings{ng1999-etal:policy,
    title={Policy Invariance Under Reward Transformations: Theory and Application to Reward Shaping},
    author={Ng, Andrew Y and Harada, Daishi and Russell, Stuart J},
    booktitle={Proceedings of the Sixteenth International Conference on Machine Learning},
    pages={278--287},
    year={1999}
}

@inproceedings{wu-etal:2023finegrained,
    title={Fine-Grained Human Feedback Gives Better Rewards for Language Model Training},
    author={Zeqiu Wu and Yushi Hu and Weijia Shi and Nouha Dziri and Alane Suhr and Prithviraj Ammanabrolu and Noah A. Smith and Mari Ostendorf and Hannaneh Hajishirzi},
    booktitle={Thirty-seventh Conference on Neural Information Processing Systems},
    year={2023}
}

@inproceedings{gao-etal:2023scaling,
    title={Scaling laws for reward model overoptimization},
    author={Gao, Leo and Schulman, John and Hilton, Jacob},
    booktitle={International Conference on Machine Learning},
    pages={10835--10866},
    year={2023},
    organization={PMLR}
}

@book{goodhart:1984problems,
    title={Problems of monetary management: the UK experience},
    author={Goodhart, Charles AE},
    year={1984},
    publisher={Springer}
}

@article{skalse-etal:2022defining,
    title={Defining and characterizing reward gaming},
    author={Skalse, Joar and Howe, Nikolaus and Krasheninnikov, Dmitrii and Krueger, David},
    journal={Advances in Neural Information Processing Systems},
    volume={35},
    pages={9460--9471},
    year={2022}
}

@inproceedings{pan-etal:2022theeffects,
   title={The Effects of Reward Misspecification: Mapping and Mitigating Misaligned Models},
   author={Alexander Pan and Kush Bhatia and Jacob Steinhardt},
   booktitle={International Conference on Learning Representations},
   year={2022}
}

@misc{krakovna-etal:2020specification,
    howpublished = {\url{https://deepmind.google/discover/blog/specification-gaming-the-flip-side-of-ai-ingenuity}},
    title = {Specification gaming: the flip side of AI ingenuity},
    author = {Victoria Krakovna and Jonathan Uesato and Vladimir Mikulik and Matthew Rahtz and Tom Everitt and Ramana Kumar and Zac Kenton and Jan Leike and Shane Legg},
    year = {2020}
}

@inproceedings{coste-etal:2024reward,
    title={Reward Model Ensembles Help Mitigate Overoptimization},
    author={Thomas Coste and Usman Anwar and Robert Kirk and David Krueger},
    booktitle={The Twelfth International Conference on Learning Representations},
    year={2024}
}

@book{miettinen:1999nonlinear,
    title={Nonlinear multiobjective optimization},
    author={Miettinen, Kaisa},
    volume={12},
    year={1999},
    publisher={Springer Science \& Business Media}
}

@article{lambert-etal:2024rewardbench,
    title={Rewardbench: Evaluating reward models for language modeling},
    author={Lambert, Nathan and Pyatkin, Valentina and Morrison, Jacob and Miranda, LJ and Lin, Bill Yuchen and Chandu, Khyathi and Dziri, Nouha and Kumar, Sachin and Zick, Tom and Choi, Yejin and Smith, Noah A. and Hajishirzi, Hannaneh},
    journal={arXiv preprint arXiv:2403.13787},
    year={2024}
}

@article{rafailov-etal:2024direct,
    title={Direct preference optimization: Your language model is secretly a reward model},
    author={Rafailov, Rafael and Sharma, Archit and Mitchell, Eric and Manning, Christopher D and Ermon, Stefano and Finn, Chelsea},
    journal={Advances in Neural Information Processing Systems},
    volume={36},
    year={2024}
}

@article{lee-etal:2023rlaif,
    title={Rlaif: Scaling reinforcement learning from human feedback with AI feedback},
    author={Lee, Harrison and Phatale, Samrat and Mansoor, Hassan and Lu, Kellie Ren and Mesnard, Thomas and Ferret, Johan and Bishop, Colton and Hall, Ethan and Carbune, Victor and Rastogi, Abhinav},
    journal={arXiv preprint arXiv:2309.00267},
    year={2023}
}

@InProceedings{cui-etal:2024ultra,
    title = 	 {{ULTRAFEEDBACK}: Boosting Language Models with Scaled {AI} Feedback},
    author =       {Cui, Ganqu and Yuan, Lifan and Ding, Ning and Yao, Guanming and He, Bingxiang and Zhu, Wei and Ni, Yuan and Xie, Guotong and Xie, Ruobing and Lin, Yankai and Liu, Zhiyuan and Sun, Maosong},
    booktitle = 	 {Proceedings of the 41st International Conference on Machine Learning},
    pages = 	 {9722--9744},
    year = 	 {2024},
    volume = 	 {235},
}

@article{dubois-etal:2024alpacafarm,
    title={Alpacafarm: A simulation framework for methods that learn from human feedback},
    author={Dubois, Yann and Li, Chen Xuechen and Taori, Rohan and Zhang, Tianyi and Gulrajani, Ishaan and Ba, Jimmy and Guestrin, Carlos and Liang, Percy S and Hashimoto, Tatsunori B},
    journal={Advances in Neural Information Processing Systems},
    volume={36},
    year={2024}
}

@article{nakano-etal:2021webgpt,
    title={Webgpt: Browser-assisted question-answering with human feedback},
    author={Reiichiro Nakano and Jacob Hilton and Suchir Balaji and Jeff Wu and Long Ouyang and Christina Kim and Christopher Hesse and Shantanu Jain and Vineet Kosaraju and William Saunders and Xu Jiang and Karl Cobbe and Tyna Eloundou and Gretchen Krueger and Kevin Button and Matthew Knight and Benjamin Chess and John Schulman},
    journal={arXiv preprint arXiv:2112.09332},
    year={2021}
}

@article{wiener:1960some,
    title={Some Moral and Technical Consequences of Automation: As machines learn they may develop unforeseen strategies at rates that baffle their programmers.},
    author={Wiener, Norbert},
    journal={Science},
    volume={131},
    number={3410},
    pages={1355--1358},
    year={1960},
    publisher={American Association for the Advancement of Science}
}

@article{bengio-etal:2024managing,
    title={Managing extreme AI risks amid rapid progress},
    author={Yoshua Bengio and Geoffrey Hinton and Andrew Yao and Dawn Song and Pieter Abbeel and Trevor Darrell and Yuval Noah Harari and Ya-Qin Zhang and Lan Xue and Shai 		Shalev-Shwartz and Gillian K. Hadfield and Jeff Clune and Tegan Maharaj and Frank Hutter and Atilim Gunes Baydin and Sheila A. McIlraith and Qiqi Gao and Ashwin Acharya and David Krueger and Anca Dragan and Philip Torr and Stuart Russell and Daniel Kahneman and Jan Markus Brauner and S{\"o}ren Mindermann},
    journal={Science},
    volume={384},
    number={6698},
    pages={842--845},
    year={2024},
    publisher={American Association for the Advancement of Science}
}

@book{russell:2019human,
    title={Human Compatible: Artificial Intelligence and the Problem of Controls},
    author={Russell, Stuart},
    publisher={Viking },
    year={2019}
}

@article{yang2024qwen2,
  title={Qwen2. 5 Technical Report},
  author={Yang, An and Yang, Baosong and Zhang, Beichen and Hui, Binyuan and Zheng, Bo and Yu, Bowen and Li, Chengyuan and Liu, Dayiheng and Huang, Fei and Wei, Haoran and others},
  journal={arXiv preprint arXiv:2412.15115},
  year={2024}
}

@article{liu2024deepseek,
  title={DeepSeek-V3 Technical Report},
  author={Liu, Aixin and Feng, Bei and Xue, Bing and Wang, Bingxuan and Wu, Bochao and Lu, Chengda and Zhao, Chenggang and Deng, Chengqi and Zhang, Chenyu and Ruan, Chong and others},
  journal={arXiv preprint arXiv:2412.19437},
  year={2024}
}

@inproceedings{mnih-etal:2016asynchronous,
    title={Asynchronous methods for deep reinforcement learning},
    author={Mnih, Volodymyr and Badia, Adri{\`a} Puigdom{\`e}nech and Mirza, Mehdi and Graves, Alex and Harley, Tim and Lillicrap, Timothy P and Silver, David and Kavukcuoglu, Koray},
    booktitle={Proceedings of the 33rd International Conference on International Conference on Machine Learning},
    pages={1928--1937},
    year={2016}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CHAPTER 12

@article{deepseek:2025r1,
    title={Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning},
    author={Deepseek},
    journal={arXiv preprint arXiv:2501.12948},
    year={2025}
}

@inproceedings{fan-etal:2018hierarchical,
    title={Hierarchical Neural Story Generation},
    author={Fan, Angela and Lewis, Mike and Dauphin, Yann},
    booktitle={Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
    pages={889--898},
    year={2018}
}

@inproceedings{holtzman-etal:2020the,
    title={The Curious Case of Neural Text Degeneration},
    author={Ari Holtzman and Jan Buys and Li Du and Maxwell Forbes and Yejin Choi},
    booktitle={International Conference on Learning Representations},
     year={2020}
}

@article{su-etal:2022contrastive,
    title={A contrastive framework for neural text generation},
    author={Su, Yixuan and Lan, Tian and Wang, Yan and Yogatama, Dani and Kong, Lingpeng and Collier, Nigel},
    journal={Advances in Neural Information Processing Systems},
    volume={35},
    pages={21548--21561},
    year={2022}
}

@misc{nvidia-nim-llm-benchmarking,
    howpublished={\url{https://docs.nvidia.com/nim/benchmarking/llm/latest/metrics.html}},
    note={Retrieved 2025-03-17},
    title={NVIDIA NIM LLMs Benchmarking},
    author={Nvidia},
    year={2025}
}

@article{wu-etal:2023fast,
    title={Fast distributed inference serving for large language models},
    author={Wu, Bingyang and Zhong, Yinmin and Zhang, Zili and Liu, Shengyu and Liu, Fangyue and Sun, Yuanhang and Huang, Gang and Liu, Xuanzhe and Jin, Xin},
    journal={arXiv preprint arXiv:2305.05920},
    year={2023}
}

@inproceedings{patel-etal:2024splitwise,
    title={Splitwise: Efficient generative llm inference using phase splitting},
    author={Patel, Pratyush and Choukse, Esha and Zhang, Chaojie and Shah, Aashaka and Goiri, {\'I}{\~n}igo and Maleki, Saeed and Bianchini, Ricardo},
    booktitle={2024 ACM/IEEE 51st Annual International Symposium on Computer Architecture (ISCA)},
    pages={118--132},
    year={2024},
    organization={IEEE}
}

@inproceedings{zhong-etal:2024distserve,
    title={$\{$DistServe$\}$: Disaggregating prefill and decoding for goodput-optimized large language model serving},
    author={Zhong, Yinmin and Liu, Shengyu and Chen, Junda and Hu, Jianbo and Zhu, Yibo and Liu, Xuanzhe and Jin, Xin and Zhang, Hao},
    booktitle={18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24)},
    pages={193--210},
    year={2024}
}

@misc{timonin-etal:2022,
    howpublished={\url{https://developer.nvidia.com/blog/accelerated-inference-for-large-transformer-models-using-nvidia-fastertransformer-and-nvidia-triton-inference-server/}},
    title={Accelerated Inference for Large Transformer Models Using NVIDIA Triton Inference Server},
    author={Denis Timonin and BoYang Hsueh and Vinh Nguyen},
    publisher={Official Nvidia Blog},
    year={2022}
}

@inproceedings{yu-ettal:2022orca,
    title={Orca: A distributed serving system for $\{$Transformer-Based$\}$ generative models},
    author={Yu, Gyeong-In and Jeong, Joo Seong and Kim, Geon-Woo and Kim, Soojeong and Chun, Byung-Gon},
    booktitle={16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22)},
    pages={521--538},
    year={2022}
}

@inproceedings{agrawal-etal:2024taming,
    title={Taming $\{$Throughput-Latency$\}$ tradeoff in $\{$LLM$\}$ inference with $\{$Sarathi-Serve$\}$},
    author={Agrawal, Amey and Kedia, Nitin and Panwar, Ashish and Mohan, Jayashree and Kwatra, Nipun and Gulavani, Bhargav and Tumanov, Alexey and Ramjee, Ramachandran},
    booktitle={18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24)},
    pages={117--134},
    year={2024}
}

@article{agrawal-etal:2023sarathi,
    title={Sarathi: Efficient llm inference by piggybacking decodes with chunked prefills},
    author={Agrawal, Amey and Panwar, Ashish and Mohan, Jayashree and Kwatra, Nipun and Gulavani, Bhargav S and Ramjee, Ramachandran},
    journal={arXiv preprint arXiv:2308.16369},
    year={2023}
}

@article{li-etal:2024llm,
    title={Llm inference serving: Survey of recent advances and opportunities},
    author={Li, Baolin and Jiang, Yankai and Gadepally, Vijay and Tiwari, Devesh},
    journal={arXiv preprint arXiv:2407.12391},
    year={2024}
}

@misc{briski:2025how,
    title={How Scaling Laws Drive Smarter, More Powerful AI},
    author={Kari Briski},
    publisher={blogs.nvidia.com},
    year={2025}
}

@article{brown-etal:2024large,
    title={Large language monkeys: Scaling inference compute with repeated sampling},
    author={Brown, Bradley and Juravsky, Jordan and Ehrlich, Ryan and Clark, Ronald and Le, Quoc V and R{\'e}, Christopher and Mirhoseini, Azalia},
    journal={arXiv preprint arXiv:2407.21787},
    year={2024}
}

@article{snell-etal:2024scaling,
    title={Scaling llm test-time compute optimally can be more effective than scaling model parameters},
    author={Snell, Charlie and Lee, Jaehoon and Xu, Kelvin and Kumar, Aviral},
    journal={arXiv preprint arXiv:2408.03314},
    year={2024}
}

@inproceedings{gou-etal:2024critic,
    title={CRITIC: Large Language Models Can Self-Correct with Tool-Interactive Critiquing},
    author={Gou, Zhibin and Shao, Zhihong and Gong, Yeyun and Yang, Yujiu and Duan, Nan and Chen, Weizhu and others},
    booktitle={The Twelfth International Conference on Learning Representations},
    year={2024}
}

@inproceedings{zhang-etal:2024small,
    title={Small Language Models Need Strong Verifiers to Self-Correct Reasoning},
    author={Zhang, Yunxiang and Khalifa, Muhammad and Logeswaran, Lajanugen and Kim, Jaekyeom and Lee, Moontae and Lee, Honglak and Wang, Lu},
    booktitle={ACL (Findings)},
    year={2024}
}

@article{shinn-etal:2023reflexion,
    title={Reflexion: Language agents with verbal reinforcement learning},
    author={Shinn, Noah and Cassano, Federico and Gopinath, Ashwin and Narasimhan, Karthik and Yao, Shunyu},
    journal={Advances in Neural Information Processing Systems},
    volume={36},
    pages={8634--8652},
    year={2023}
}

@inproceedings{snell-etal:2025scaling,
    title={Scaling {LLM} Test-Time Compute Optimally Can be More Effective than Scaling Parameters for Reasoning},
    author={Charlie Victor Snell and Jaehoon Lee and Kelvin Xu and Aviral Kumar},
    booktitle={The Thirteenth International Conference on Learning Representations},
    year={2025}
}

@article{viterbi:1967error,
    title={Error bounds for convolutional codes and an asymptotically optimum decoding algorithm},
    author={Viterbi, Andrew J},
    journal={IEEE Transactions on Information Theory},
    year={1967}
}

@book{wozengraft-and-reiffen:1961,
    title={Sequential Decoding},
    author={ John M. Wozengraft and Barney Reiffen},
    publisher={The MIT Press},
    year={1961}
}

@article{forney-etal:1972maximum,
    title={Maximum-likelihood sequence estimation of digital sequences in the presence of intersymbol interference},
  author={Forney, GDJR},
    journal={IEEE Transactions on Information theory},
    volume={18},
    number={3},
    pages={363--378},
    year={1972},
    publisher={IEEE}
}

@inproceedings{houlsby-etal:2019adapter,
  title={Parameter-Efficient Transfer Learning for NLP},
  author={Houlsby, Neil and Giurgiu, Andrei and Jastrzebski, Stanislaw and Morrone, Bruna and de Laroussilhe, Quentin and Gesmundo, Andrea and Attariyan, Mona and Gelly, Sylvain},
  booktitle={International Conference on Machine Learning (ICML)},
  year={2019}
}

@inproceedings{lester-etal:2021prompt,
  title={The Power of Scale for Parameter-Efficient Prompt Tuning},
  author={Lester, Brian and Al-Rfou, Rami and Constant, Noah},
  booktitle={Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
  year={2021}
}

@inproceedings{hu-etal:2021lora,
  title={LoRA: Low-Rank Adaptation of Large Language Models},
  author={Hu, Edward J and Shen, Yelong and Wallis, Phillip and Allen-Zhu, Zeyuan and Li, Yuanzhi and Wang, Shean and Wang, Lu and Chen, Weizhu},
  booktitle={International Conference on Learning Representations (ICLR)},
  year={2021}
}

@inproceedings{dettmers-etal:2023qlora,
  title={QLoRA: Efficient Finetuning of Quantized LLMs},
  author={Dettmers, Tim and Pagnoni, Artidoro and Holtzman, Ari and Zettlemoyer, Luke},
  booktitle={Advances in Neural Information Processing Systems (NeurIPS)},
  year={2023}
}

@inproceedings{liu-etal:2024dora,
  title={DoRA: Weight-Decomposed Low-Rank Adaptation},
  author={Liu, Shih-Yang and Wang, Chien-Yi and Yin, Hongxu and Molchanov, Pavlo and Wang, Yu-Chiang Frank and Cheng, Kwang-Ting and Chen, Min-Hung},
  booktitle={Proceedings of the 41st International Conference on Machine Learning (ICML)},
  year={2024}
}

@inproceedings{li-etal:2018intrinsic,
  title={Measuring the Intrinsic Dimension of Objective Landscapes},
  author={Li, Chunyuan and Farkhoor, Heerad and Liu, Rosanne and Yosinski, Jason},
  booktitle={International Conference on Learning Representations (ICLR)},
  year={2018}
}

@book{peirce1931:collected,
  title={Collected Papers of Charles Sanders Peirce},
  author={Peirce, Charles Sanders},
  editor={Hartshorne, Charles and Weiss, Paul},
  volume={1-6},
  year={1931},
  publisher={Harvard University Press}
}

@inproceedings{yao-etal:2023react,
  title={ReAct: Synergizing Reasoning and Acting in Language Models},
  author={Yao, Shunyu and Zhao, Jeffrey and Yu, Dian and Du, Nan and Shafran, Izhak and Narasimhan, Karthik and Cao, Yuan},
  booktitle={International Conference on Learning Representations (ICLR)},
  year={2023}
}

@article{wang-etal:2024survey,
  title={A Survey on Large Language Model based Autonomous Agents},
  author={Wang, Lei and Ma, Chen and Feng, Xueyang and Zhang, Zeyu and Yang, Hao and Zhang, Jingsen and Chen, Zhiyuan and Tang, Ji and Chen, Xu and Lin, Yankai and others},
  journal={Frontiers of Computer Science},
  volume={18},
  number={5},
  pages={185345},
  year={2024},
  publisher={Springer}
}

@article{yang2025qwen3,
  title   = {Qwen3 Technical Report},
  author  = {Qwen Team},
  journal = {arXiv preprint arXiv:2505.09388},
  year    = {2025}
}

@misc{mistral2025mistral3,
  title        = {Introducing Mistral 3},
  author       = {{Mistral AI}},
  year         = {2025},
  howpublished = {\url{https://mistral.ai/news/mistral-3}},
  note         = {Accessed: 2026-07-28}
}

@article{gemmateam2025gemma3,
  title   = {Gemma 3 Technical Report},
  author  = {Gemma Team},
  journal = {arXiv preprint arXiv:2503.19786},
  year    = {2025}
}

@book{edwards:1994historical,
    title={The historical development of the calculus},
    author={Edwards, CH Jr},
    year={1994},
    publisher={Springer Science \& Business Media}
}

@book{dunham:2005calculus,
    title={The calculus gallery: Masterpieces from Newton to Lebesgue},
    author={Dunham, William},
    year={2005},
    publisher={Princeton University Press}
}

@book{apostol:1991calculus,
    title={Calculus, Volume 1},
    author={Apostol, Tom M},
    year={1991},
    publisher={John Wiley \& Sons}
}

@book{spivak:2006calculus,
    title={Calculus},
    author={Spivak, Michael},
    year={2006},
    publisher={Cambridge University Press}
}

@book{hartman:2002ordinary,
    title={Ordinary differential equations},
    author={Hartman, Philip},
    year={2002},
    publisher={SIAM}
}

@book{hirsch-etal:2013differential,
    title={Differential equations, dynamical systems, and an introduction to chaos},
    author={Hirsch, Morris W and Smale, Stephen and Devaney, Robert L},
    year={2013},
    publisher={Academic press}
}

@book{stoer-etal:1980introduction,
    title={Introduction to numerical analysis},
    author={Stoer, Josef and Bulirsch, Roland and Bartels, R and Gautschi, Walter and Witzgall, Christoph},
    year={1980},
    publisher={Springer}
}

@book{richard:2015numerical,
    title={Numerical Analysis},
    author={ Richard L. Burden and J. Douglas Faires and Annette M. Burden},
    year={2015},
    publisher={Cengage Learning}
}

@article{qian:1999momentum,
    title={On the momentum term in gradient descent learning algorithms},
    author={Qian, Ning},
    journal={Neural networks},
    volume={12},
    number={1},
    pages={145--151},
    year={1999},
    publisher={Elsevier}
}

@article{su-etal:2016differential,
    title={A differential equation for modeling Nesterov's accelerated gradient method: Theory and insights},
    author={Su, Weijie and Boyd, Stephen and Candes, Emmanuel J},
    journal={Journal of Machine Learning Research},
    volume={17},
    number={153},
    pages={1--43},
    year={2016}
}

@book{norris:1998markov,
    title={Markov chains},
    author={Norris, James R},
    year={1998},
    publisher={Cambridge university press}
}

@article{vaswani-etal:2017attention,
    title={Attention is all you need},
    author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
    journal={Advances in neural information processing systems},
    volume={30},
    year={2017}
}

@article{bai-etal:2023qwen,
    title={Qwen technical report},
    author={Jinze Bai and Shuai Bai and Yunfei Chu and Zeyu Cui and Kai Dang and Xiaodong Deng and Yang Fan and Wenbin Ge and Yu Han and Fei Huang and Binyuan Hui and Luo Ji and Mei Li and Junyang Lin and Runji Lin and Dayiheng Liu and Gao Liu and Chengqiang Lu and Keming Lu and Jianxin Ma and Rui Men and Xingzhang Ren and Xuancheng Ren and Chuanqi Tan and Sinan Tan and Jianhong Tu and Peng Wang and Shijie Wang and Wei Wang and Shengguang Wu and Benfeng Xu and Jin Xu and An Yang and Hao Yang and Jian Yang and Shusheng Yang and Yang Yao and Bowen Yu and Hongyi Yuan and Zheng Yuan and Jianwei Zhang and Xingxuan Zhang and Yichang Zhang and Zhenru Zhang and Chang Zhou and Jingren Zhou and Xiaohuan Zhou and Tianhang Zhu},
    journal={arXiv preprint arXiv:2309.16609},
    year={2023}
}

@article{liu-etal:2024deepseek,
    title={Deepseek-v3 technical report},
    author={Aixin Liu and Bei Feng and Bing Xue and Bingxuan Wang and Bochao Wu and Chengda Lu and Chenggang Zhao and Chengqi Deng and Chenyu Zhang and Chong Ruan and Damai Dai and Daya Guo and Dejian Yang and Deli Chen and Dongjie Ji and Erhang Li and Fangyun Lin and Fucong Dai and Fuli Luo and Guangbo Hao and Guanting Chen and Guowei Li and H. Zhang and Han Bao and Hanwei Xu and Haocheng Wang and Haowei Zhang and Honghui Ding and Huajian Xin and Huazuo Gao and Hui Li and Hui Qu and J. L. Cai and Jian Liang and Jianzhong Guo and Jiaqi Ni and Jiashi Li and Jiawei Wang and Jin Chen and Jingchang Chen and Jingyang Yuan and Junjie Qiu and Junlong Li and Junxiao Song and Kai Dong and Kai Hu and Kaige Gao and Kang Guan and Kexin Huang and Kuai Yu and Lean Wang and Lecong Zhang and Lei Xu and Leyi Xia and Liang Zhao and Litong Wang and Liyue Zhang and Meng Li and Miaojun Wang and Mingchuan Zhang and Minghua Zhang and Minghui Tang and Mingming Li and Ning Tian and Panpan Huang and Peiyi Wang and Peng Zhang and Qiancheng Wang and Qihao Zhu and Qinyu Chen and Qiushi Du and R. J. Chen and R. L. Jin and Ruiqi Ge and Ruisong Zhang and Ruizhe Pan and Runji Wang and Runxin Xu and Ruoyu Zhang and Ruyi Chen and S. S. Li and Shanghao Lu and Shangyan Zhou and Shanhuang Chen and Shaoqing Wu and Shengfeng Ye and Shengfeng Ye and Shirong Ma and Shiyu Wang and Shuang Zhou and Shuiping Yu and Shunfeng Zhou and Shuting Pan and T. Wang and Tao Yun and Tian Pei and Tianyu Sun and W. L. Xiao and Wangding Zeng and Wanjia Zhao and Wei An and Wen Liu and Wenfeng Liang and Wenjun Gao and Wenqin Yu and Wentao Zhang and X. Q. Li and Xiangyue Jin and Xianzu Wang and Xiao Bi and Xiaodong Liu and Xiaohan Wang and Xiaojin Shen and Xiaokang Chen and Xiaokang Zhang and Xiaosha Chen and Xiaotao Nie and Xiaowen Sun and Xiaoxiang Wang and Xin Cheng and Xin Liu and Xin Xie and Xingchao Liu and Xingkai Yu and Xinnan Song and Xinxia Shan and Xinyi Zhou and Xinyu Yang and Xinyuan Li and Xuecheng Su and Xuheng Lin and Y. K. Li and Y. Q. Wang and Y. X. Wei and Y. X. Zhu and Yang Zhang and Yanhong Xu and Yanhong Xu and Yanping Huang and Yao Li and Yao Zhao and Yaofeng Sun and Yaohui Li and Yaohui Wang and Yi Yu and Yi Zheng and Yichao Zhang and Yifan Shi and Yiliang Xiong and Ying He and Ying Tang and Yishi Piao and Yisong Wang and Yixuan Tan and Yiyang Ma and Yiyuan Liu and Yongqiang Guo and Yu Wu and Yuan Ou and Yuchen Zhu and Yuduan Wang and Yue Gong and Yuheng Zou and Yujia He and Yukun Zha and Yunfan Xiong and Yunxian Ma and Yuting Yan and Yuxiang Luo and Yuxiang You and Yuxuan Liu and Yuyang Zhou and Z. F. Wu and Z. Z. Ren and Zehui Ren and Zhangli Sha and Zhe Fu and Zhean Xu and Zhen Huang and Zhen Zhang and Zhenda Xie and Zhengyan Zhang and Zhewen Hao and Zhibin Gou and Zhicheng Ma and Zhigang Yan and Zhihong Shao and Zhipeng Xu and Zhiyu Wu and Zhongyu Zhang and Zhuoshu Li and Zihui Gu and Zijia Zhu and Zijun Liu and Zilin Li and Ziwei Xie and Ziyang Song and Ziyi Gao and Zizheng Pan},
    journal={arXiv preprint arXiv:2412.19437},
    year={2024}
}

@inproceedings{baevski-etal:2019adaptive,
    title={Adaptive Input Representations for Neural Language Modeling},
    author={Alexei Baevski and Michael Auli},
    booktitle={International Conference on Learning Representations},
    year={2019}
}

@inproceedings{lu-etal:2020understanding,
    title={Understanding and Improving Transformer From a Multi-Particle Dynamic System Point of View.},
    author={Lu, Yiping and Li, Zhuohan and He, Di and Sun, Zhiqing and Dong, Bin and Qin, Tao and Wang, Liwei and Liu, Tie-yan},
    booktitle={ICLR 2020 Workshop on Integration of Deep Neural Models and Differential Equations},
    year={2020}
}

@article{strang:1968construction,
    title={On the construction and comparison of difference schemes},
    author={Strang, Gilbert},
    journal={SIAM journal on numerical analysis},
    volume={5},
    number={3},
    pages={506--517},
    year={1968},
    publisher={SIAM}
}

@article{li-etal:2024predictor,
    title={Predictor-corrector enhanced transformers with exponential moving average coefficient learning},
    author={Li, Bei and Zheng, Tong and Wang, Rui and Liu, Jiahao and Guo, Junliang and Tan, Xu and Xiao, Tong and Zhu, JingBo and Wang, Jingang and Cai, Xunliang},
    journal={Advances in Neural Information Processing Systems},
    volume={37},
    pages={20358--20382},
    year={2024}
}

@inproceedings{liu-etal:2025iiet,
    title={IIET: Efficient Numerical Transformer via Implicit Iterative Euler Method},
    author={Liu, Xinyu and Li, Bei and Liu, Jiahao and Ruan, Junhao and Jiao, Kechen and Tang, Hongyin and Wang, Jingang and Xiao, Tong and Zhu, Jingbo},
    booktitle={Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing},
    pages={8955--8969},
    year={2025}
}

@book{hairer-and-wanner:1996solving,
    title={Solving Ordinary Differential Equations II},
    author={Hairer, Ernst and Wanner, Gerhard},
    year={1996},
    publisher={Cambridge university press}
}

@article{neyshabur:2017implicit,
    title={Implicit regularization in deep learning},
    author={Neyshabur, Behnam},
    journal={arXiv preprint arXiv:1709.01953},
    year={2017}
}

@inproceedings{miyato-etal:2018spectral,
    title={Spectral Normalization for Generative Adversarial Networks},
    author={Miyato, Takeru and Kataoka, Toshiki and Koyama, Masanori and Yoshida, Yuichi},
    booktitle={International Conference on Learning Representations},
    year={2018}
}

@article{gouk-etal:2021regularisation,
    title={Regularisation of neural networks by enforcing lipschitz continuity},
    author={Gouk, Henry and Frank, Eibe and Pfahringer, Bernhard and Cree, Michael J},
    journal={Machine Learning},
    volume={110},
    number={2},
    pages={393--416},
    year={2021},
    publisher={Springer}
}

@article{haber-and-ruthotto:2017stable,
    title={Stable architectures for deep neural networks},
    author={Haber, Eldad and Ruthotto, Lars},
    journal={Inverse problems},
    volume={34},
    number={1},
    year={2017},
    publisher={IOP Publishing}
}

@article{coddington:1955theory,
    title={Theory of Ordinary Differential Equations},
    author={Coddington, EA},
    journal={McGraw-Hill Book Company},
    year={1955}
}

@book{tao:2011introduction,
    title={An introduction to measure theory},
    author={Tao, Terence},
    volume={126},
    year={2011},
    publisher={American Mathematical Society}
}

@article{hoogeboom-etal:2019integer,
    title={Integer discrete flows and lossless compression},
    author={Hoogeboom, Emiel and Peters, Jorn and Van Den Berg, Rianne and Welling, Max},
    journal={Advances in Neural Information Processing Systems},
    volume={32},
    year={2019}
}

@article{tran2019-etal:discrete,
    title={Discrete flows: Invertible generative models of discrete data},
    author={Tran, Dustin and Vafa, Keyon and Agrawal, Kumar and Dinh, Laurent and Poole, Ben},
    journal={Advances in Neural Information Processing Systems},
    volume={32},
    year={2019}
}

@article{dinh-etal:2014nice,
    title={Nice: Non-linear independent components estimation},
    author={Dinh, Laurent and Krueger, David and Bengio, Yoshua},
    journal={arXiv preprint arXiv:1410.8516},
    year={2014}
}

@inproceedings{dinh-etal:2017density,
    title={Density estimation using Real NVP},
    author={Dinh, Laurent and Sohl-Dickstein, Jascha and Bengio, Samy},
    booktitle={International Conference on Learning Representations},
    year={2017}
}

@article{papamakarios-etal:2017masked,
    title={Masked autoregressive flow for density estimation},
    author={Papamakarios, George and Pavlakou, Theo and Murray, Iain},
    journal={Advances in neural information processing systems},
    volume={30},
    year={2017}
}

@article{kingma-etal:2016improved,
    title={Improved variational inference with inverse autoregressive flow},
    author={Kingma, Durk P and Salimans, Tim and Jozefowicz, Rafal and Chen, Xi and Sutskever, Ilya and Welling, Max},
    journal={Advances in neural information processing systems},
    volume={29},
    year={2016}
}

@article{kingma-and-dhariwal:2018glow,
    title={Glow: Generative flow with invertible 1x1 convolutions},
    author={Kingma, Durk P and Dhariwal, Prafulla},
    journal={Advances in neural information processing systems},
    volume={31},
    year={2018}
}

@inproceedings{behrmann-etal:2019invertible,
    title={Invertible residual networks},
    author={Behrmann, Jens and Grathwohl, Will and Chen, Ricky TQ and Duvenaud, David and Jacobsen, J{\"o}rn-Henrik},
    booktitle={International conference on machine learning},
    pages={573--582},
    year={2019},
    organization={PMLR}
}

@article{chen-etal:2019residual,
    title={Residual flows for invertible generative modeling},
    author={Chen, Ricky TQ and Behrmann, Jens and Duvenaud, David K and Jacobsen, J{\"o}rn-Henrik},
    journal={Advances in neural information processing systems},
    volume={32},
    year={2019}
}

@book{batchelor:2000introduction,
    title={An introduction to fluid dynamics},
    author={Batchelor, George Keith},
    year={2000},
    publisher={Cambridge university press}
}

@article{croitoru-etal:2023diffusion,
    title={Diffusion models in vision: A survey},
    author={Croitoru, Florinel-Alin and Hondru, Vlad and Ionescu, Radu Tudor and Shah, Mubarak},
    journal={IEEE transactions on pattern analysis and machine intelligence},
    volume={45},
    number={9},
    pages={10850--10869},
    year={2023},
    publisher={IEEE}
}

@article{yang-etal:2023diffusion,
    title={Diffusion models: A comprehensive survey of methods and applications},
    author={Yang, Ling and Zhang, Zhilong and Song, Yang and Hong, Shenda and Xu, Runsheng and Zhao, Yue and Zhang, Wentao and Cui, Bin and Yang, Ming-Hsuan},
    journal={ACM computing surveys},
    volume={56},
    number={4},
    pages={1--39},
    year={2023},
    publisher={ACM New York, NY, USA}
}

@misc{song:2021generative,
    author = {Yang Song},
    title = {Generative Modeling by Estimating Gradients of the Data Distribution},
    year = {2021},
    url = {https://yang-song.net/blog/2021/score/}
}

@misc{dieleman:2023perspectives,
    author = {Sander Dieleman},
    title = {Perspectives on diffusion},
    year = {2023},
    url = {https://sander.ai/2023/07/20/perspectives.html}
}

@inproceedings{kingma-and-gao:2023understanding,
    title={Understanding Diffusion Objectives as the {ELBO} with Simple Data Augmentation},
    author={Diederik P Kingma and Ruiqi Gao},
    booktitle={Thirty-seventh Conference on Neural Information Processing Systems},
    year={2023}
}

@article{luo:2022understand,
    title={Understanding Diffusion Models: A Unified Perspective},
    author={Calvin Luo},
    year={2022},
    journal={arXiv}
}

@article{kingma-and-welling:2019introduction,
    title={An introduction to variational autoencoders},
    author={Kingma, Diederik P and Welling, Max},
    journal={Foundations and Trends{\textregistered} in Machine Learning},
    volume={12},
    number={4},
    pages={307--392},
    year={2019},
    publisher={Now Publishers}
}

@article{kingma-and-welling:2013auto,
    title={Auto-encoding variational bayes},
    author={Kingma, Diederik P and Welling, Max},
    journal={arXiv preprint arXiv:1312.6114},
    year={2013}
}

@article{bank-etal:2023autoencoders,
    title={Autoencoders},
    author={Bank, Dor and Koenigstein, Noam and Giryes, Raja},
    journal={Machine learning for data science handbook: data mining and knowledge discovery handbook},
    pages={353--374},
    year={2023},
    publisher={Springer}
}

@book{oksendal:2013stochastic,
    title={Stochastic differential equations: an introduction with applications},
    author={Oksendal, Bernt},
    year={2013},
    publisher={Springer Science \& Business Media}
}

@book{karatzas-and-shreve:2014brownian,
    title={Brownian motion and stochastic calculus},
    author={Karatzas, Ioannis and Shreve, Steven},
    year={2014},
    publisher={springer}
}

@article{anderson:1982reverse,
    title = {Reverse-time diffusion equation models},
    journal = {Stochastic Processes and their Applications},
    volume = {12},
    number = {3},
    pages = {313-326},
    year = {1982},
    author = {Brian D.O. Anderson}
}

@article{hyvarinen-and-dayan:2005estimation,
    title={Estimation of non-normalized statistical models by score matching.},
    author={Hyv{\"a}rinen, Aapo and Dayan, Peter},
    journal={Journal of Machine Learning Research},
    volume={6},
    number={4},
    year={2005}
}

@article{vincent:2011connection,
    title={A connection between score matching and denoising autoencoders},
    author={Vincent, Pascal},
    journal={Neural computation},
    volume={23},
    number={7},
    pages={1661--1674},
    year={2011},
    publisher={MIT Press}
}

@article{kloeden-and-pearson:1977numerical,
    title={The numerical solution of stochastic differential equations},
    author={Kloeden, Peter E and Pearson, RA},
    journal={The ANZIAM Journal},
    volume={20},
    number={1},
    pages={8--12},
    year={1977},
    publisher={Cambridge University Press}
}

@article{song-and-ermon:2019generative,
    title={Generative modeling by estimating gradients of the data distribution},
    author={Song, Yang and Ermon, Stefano},
    journal={Advances in neural information processing systems},
    volume={32},
    year={2019}
}

@article{ho-etal:2020denoising,
    title={Denoising diffusion probabilistic models},
    author={Ho, Jonathan and Jain, Ajay and Abbeel, Pieter},
    journal={Advances in neural information processing systems},
    volume={33},
    pages={6840--6851},
    year={2020}
}

@inproceedings{song-etal:2021scorebased,
    title={Score-Based Generative Modeling through Stochastic Differential Equations},
    author={Yang Song and Jascha Sohl-Dickstein and Diederik P Kingma and Abhishek Kumar and Stefano Ermon and Ben Poole},
    booktitle={International Conference on Learning Representations},
    year={2021}
}

@book{kloeden-and-platen:1992numerical,
    title={Numerical Solution of Stochastic Differential Equations},
    author={Peter E. Kloeden and Eckhard Platen},
    year={1992},
    publisher={Springer Berlin}
}

@book{robert-etal:1999monte,
    title={Monte Carlo statistical methods},
    author={Robert, Christian P and Casella, George and Casella, George},
    volume={2},
    year={1999},
    publisher={Springer}
}

@inproceedings{lipman-etal:2023flow,
    title={Flow Matching for Generative Modeling},
    author={Yaron Lipman and Ricky T. Q. Chen and Heli Ben-Hamu and Maximilian Nickel and Matthew Le},
    booktitle={The Eleventh International Conference on Learning Representations },
    year={2023}
}

@inproceedings{liu-etal:2023flow,
    title={Flow Straight and Fast: Learning to Generate and Transfer Data with Rectified Flow},
    author={Xingchao Liu and Chengyue Gong and qiang liu},
    booktitle={The Eleventh International Conference on Learning Representations },
    year={2023}
}

@inproceedings{albergo-etal:2023building,
    title={Building Normalizing Flows with Stochastic Interpolants},
    author={Michael Samuel Albergo and Eric Vanden-Eijnden},
    booktitle={The Eleventh International Conference on Learning Representations },
    year={2023}
}

@article{lu-etal:2022dpm,
    title={DPM-solver: A fast ode solver for diffusion probabilistic model sampling in around 10 steps},
    author={Lu, Cheng and Zhou, Yuhao and Bao, Fan and Chen, Jianfei and Li, Chongxuan and Zhu, Jun},
    journal={Advances in neural information processing systems},
    volume={35},
    pages={5775--5787},
    year={2022}
}

@article{lu-etal:2025dpmpp,
    title={Dpm-solver++: Fast solver for guided sampling of diffusion probabilistic models},
    author={Lu, Cheng and Zhou, Yuhao and Bao, Fan and Chen, Jianfei and Li, Chongxuan and Zhu, Jun},
    journal={Machine Intelligence Research},
    pages={1--22},
    year={2025},
    publisher={Springer}
}

@article{ren-etal:2025fast,
    title={Fast solvers for discrete diffusion models: Theory and applications of high-order algorithms},
    author={Ren, Yinuo and Chen, Haoxuan and Zhu, Yuchen and Guo, Wei and Chen, Yongxin and Rotskoff, Grant M and Tao, Molei and Ying, Lexing},
    journal={arXiv preprint arXiv:2502.00234},
    year={2025}
}

@inproceedings{zhou-etal:2024fast,
    title={Fast ode-based sampling for diffusion models in around 5 steps},
    author={Zhou, Zhenyu and Chen, Defang and Wang, Can and Chen, Chun},
    booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
    pages={7777--7786},
    year={2024}
}

@inproceedings{zhang-and-chenl:2022fast,
    title={Fast Sampling of Diffusion Models with Exponential Integrator},
    author={Qinsheng Zhang and Yongxin Chen},
    booktitle={NeurIPS 2022 Workshop on Score-Based Methods},
    year={2022}
}

@article{zhao-etal:2023unipc,
    title={Unipc: A unified predictor-corrector framework for fast sampling of diffusion models},
    author={Zhao, Wenliang and Bai, Lujia and Rao, Yongming and Zhou, Jie and Lu, Jiwen},
    journal={Advances in Neural Information Processing Systems},
    volume={36},
    pages={49842--49869},
    year={2023}
}

@inproceedings{liu-etal:2022pseudo,
    title={Pseudo Numerical Methods for Diffusion Models on Manifolds},
    author={Luping Liu and Yi Ren and Zhijie Lin and Zhou Zhao},
    booktitle={International Conference on Learning Representations},
    year={2022}
}

@article{karras-etal:2022elucidating,
    title={Elucidating the design space of diffusion-based generative models},
    author={Karras, Tero and Aittala, Miika and Aila, Timo and Laine, Samuli},
    journal={Advances in neural information processing systems},
    volume={35},
    pages={26565--26577},
    year={2022}
}

@article{zhang-etal:2025:flow,
    title={Flow Straighter and Faster: Efficient One-Step Generative Modeling via MeanFlow on Rectified Trajectories},
    author={Xinxi Zhang and Shiwei Tan and Quang Nguyen and Quan Dao and Ligong Han and Xiaoxiao He and Tunyu Zhang and Alen Mrdovic and Dimitris Metaxas},
    year={2025},
    journal={arXiv preprint arXiv:2511.23342}
}

@inproceedings{yang-etal:2025text,
    title={Text-to-Image Rectified Flow as Plug-and-Play Priors},
    author={Xiaofeng Yang and Chen Cheng and Xulei Yang and Fayao Liu and Guosheng Lin},
    booktitle={The Thirteenth International Conference on Learning Representations},
    year={2025}
}

@inproceedings{esser-etal:2024scaling,
    title={Scaling rectified flow transformers for high-resolution image synthesis},
    author={Patrick Esser and Sumith Kulal and Andreas Blattmann and Rahim Entezari and Jonas M{\" u}ller and Harry Saini and Yam Levi and Dominik Lorenz and Axel Sauer and Frederic Boesel and Dustin Podell and Tim Dockhorn and Zion English and Kyle Lacey and Alex Goodwin and Yannik Marek and Robin Rombach},
    booktitle={Forty-first international conference on machine learning},
    year={2024}
}

@article{greenberg-etal:2025demystifying,
    title={Demystifying flux architecture},
    author={Greenberg, Or},
    journal={arXiv preprint arXiv:2507.09595},
    year={2025}
}

@inproceedings{yan-etal:2024perflow,
    title={Pe{RF}low: Piecewise Rectified Flow as Universal Plug-and-Play Accelerator},
    author={Hanshu Yan and Xingchao Liu and Jiachun Pan and Jun Hao Liew and qiang liu and Jiashi Feng},
    booktitle={The Thirty-eighth Annual Conference on Neural Information Processing Systems},
    year={2024}
}

@article{saini-etal:2025rectified,
    title={Rectified-CFG++ for Flow Based Models},
    author={Saini, Shreshth and Gupta, Shashank and Bovik, Alan C},
    journal={arXiv preprint arXiv:2510.07631},
    year={2025}
}

@inproceedings{song-etal:2023consistency,
    title={Consistency Models},
    author={Song, Yang and Dhariwal, Prafulla and Chen, Mark and Sutskever, Ilya},
    booktitle={International Conference on Machine Learning},
    pages={32211--32252},
    year={2023},
    organization={PMLR}
}

@inproceedings{sauer-etal:2024adversarial,
    title={Adversarial diffusion distillation},
    author={Sauer, Axel and Lorenz, Dominik and Blattmann, Andreas and Rombach, Robin},
    booktitle={European Conference on Computer Vision},
    pages={87--103},
    year={2024},
    organization={Springer}
}

@article{luo-etal:2023latent,
    title={Latent consistency models: Synthesizing high-resolution images with few-step inference},
    author={Luo, Simian and Tan, Yiqin and Huang, Longbo and Li, Jian and Zhao, Hang},
    journal={arXiv preprint arXiv:2310.04378},
    year={2023}
}

@article{luo-etal:2023lcm,
    title={Lcm-lora: A universal stable-diffusion acceleration module},
    author={Luo, Simian and Tan, Yiqin and Patil, Suraj and Gu, Daniel and Von Platen, Patrick and Passos, Apolin{\' a}rio and Huang, Longbo and Li, Jian and Zhao, Hang},
    journal={arXiv preprint arXiv:2311.05556},
    year={2023}
}

@article{zheng-etal:2024trajectory,
    title={Trajectory consistency distillation},
    author={Zheng, Jianbin and Hu, Minghui and Fan, Zhongyi and Wang, Chaoyue and Ding, Changxing and Tao, Dacheng and Cham, Tat-Jen},
    journal={CoRR},
    year={2024}
}

@article{wang-etal:2024phased,
    title={Phased consistency models},
    author={Wang, Fu-Yun and Huang, Zhaoyang and Bergman, Alexander and Shen, Dazhong and Gao, Peng and Lingelbach, Michael and Sun, Keqiang and Bian, Weikang and Song, Guanglu and Liu, Yu and others},
    journal={Advances in neural information processing systems},
    volume={37},
    pages={83951--84009},
    year={2024}
}

@inproceedings{lu-and-song:2025simplifying,
    title={Simplifying, Stabilizing and Scaling Continuous-time Consistency Models},
    author={Cheng Lu and Yang Song},
    booktitle={The Thirteenth International Conference on Learning Representations},
    year={2025}
}

@article{shen-etal:2025efficient,
    title={Efficient diffusion models: A survey},
    author={Hui Shen and Jingxuan Zhang and Boning Xiong and Rui Hu and Shoufa Chen and Zhongwei Wan and Xin Wang and Yu Zhang and Zixuan Gong and Guangyin Bao and Chaofan Tao and Yongfeng Huang and Ye Yuan and Mi Zhang},
   journal={arXiv preprint arXiv:2502.06805},
    year={2025}
}

@article{li-etal:2022diffusion,
    title={Diffusion-lm improves controllable text generation},
    author={Li, Xiang and Thickstun, John and Gulrajani, Ishaan and Liang, Percy S and Hashimoto, Tatsunori B},
    journal={Advances in neural information processing systems},
    volume={35},
    pages={4328--4343},
    year={2022}
}

@article{strudel-etal:2022self,
    title={Self-conditioned embedding diffusion for text generation},
    author={Strudel, Robin and Tallec, Corentin and Altch{\'e}, Florent and Du, Yilun and Ganin, Yaroslav and Mensch, Arthur and Grathwohl, Will and Savinov, Nikolay and Dieleman, Sander and Sifre, Laurent and Leblond, R{\' e}mi},
    journal={arXiv preprint arXiv:2211.04236},
    year={2022}
}

@inproceedings{gong-etal:2023diffuseq,
    title={DiffuSeq: Sequence to Sequence Text Generation with Diffusion Models},
    author={Shansan Gong and Mukai Li and Jiangtao Feng and Zhiyong Wu and Lingpeng Kong},
    booktitle={The Eleventh International Conference on Learning Representations },
    year={2023}
}

@inproceedings{mikolov-etal:2013efficient,
    title={Efficient Estimation of Word Representations in Vector Space},
    author={Mikolov, Tomas and Chen, Kai and Corrado, Greg and Dean, Jeffrey},
    booktitle={Proceedings of the International Conference on Learning Representations (ICLR 2013)},
    year={2013}
}

@inproceedings{mikolov-etal:2013distributed,
    author = {Mikolov, Tomas and Sutskever, Ilya and Chen, Kai and Corrado, Greg and Dean, Jeffrey},
    title = {Distributed Representations of Words and Phrases and Their Compositionality},
    year = {2013},
    booktitle = {Proceedings of the 26th International Conference on Neural Information Processing Systems - Volume 2},
    pages = {3111-3119}
}

@inproceedings{rombach-etal:2022high,
    title={High-resolution image synthesis with latent diffusion models},
    author={Rombach, Robin and Blattmann, Andreas and Lorenz, Dominik and Esser, Patrick and Ommer, Bj{\"o}rn},
    booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
    pages={10684--10695},
    year={2022}
}

@article{barrault-etal:2024large,
    title={Large concept models: Language modeling in a sentence representation space},
    author={Barrault, Lo{\"\i}c and Duquenne, Paul-Ambroise and Elbayad, Maha and Kozhevnikov, Artyom and Alastruey, Belen and Andrews, Pierre and Coria, Mariano and Couairon, Guillaume and Costa-juss{\`a}, Marta R and Dale, David and others},
    journal={arXiv preprint arXiv:2412.08821},
    year={2024}
}

@article{xiao-etal:2023survey,
    title={A survey on non-autoregressive generation for neural machine translation and beyond},
    author={Xiao, Yisheng and Wu, Lijun and Guo, Junliang and Li, Juntao and Zhang, Min and Qin, Tao and Liu, Tie-yan},
    journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},
    volume={45},
    number={10},
    pages={11407--11427},
    year={2023},
    publisher={IEEE}
}

@misc{li-etal:2025fixed,
      title={Beyond Fixed: Training-Free Variable-Length Denoising for Diffusion Large Language Models},
      author={Jinsong Li and Xiaoyi Dong and Yuhang Zang and Yuhang Cao and Jiaqi Wang and Dahua Lin},
      year={2025},
      eprint={2508.00819},
      archivePrefix={arXiv}
}

@misc{yang-etal:2025diffusion,
      title={Diffusion LLM with Native Variable Generation Lengths: Let [EOS] Lead the Way},
      author={Yicun Yang and Cong Wang and Shaobo Wang and Zichen Wen and Biqing Qi and Hanlin Xu and Linfeng Zhang},
      year={2025},
      eprint={2510.24605},
      archivePrefix={arXiv}
}

@article{austin-etal:2021structured,
    title={Structured denoising diffusion models in discrete state-spaces},
    author={Austin, Jacob and Johnson, Daniel D and Ho, Jonathan and Tarlow, Daniel and Van Den Berg, Rianne},
    journal={Advances in neural information processing systems},
    volume={34},
    pages={17981--17993},
    year={2021}
}

@article{sahoo-etal:2024simple,
    title={Simple and effective masked diffusion language models},
    author={Sahoo, Subham and Arriola, Marianne and Schiff, Yair and Gokaslan, Aaron and Marroquin, Edgar and Chiu, Justin and Rush, Alexander and Kuleshov, Volodymyr},
    journal={Advances in Neural Information Processing Systems},
    volume={37},
    pages={130136--130184},
    year={2024}
}

@article{nie-etal:2025large,
    title={Large language diffusion models},
    author={Nie, Shen and Zhu, Fengqi and You, Zebin and Zhang, Xiaolu and Ou, Jingyang and Hu, Jun and Zhou, Jun and Lin, Yankai and Wen, Ji-Rong and Li, Chongxuan},
    journal={arXiv preprint arXiv:2502.09992},
    year={2025}
}

@article{zhang-etal:2025corrective,
    title={Corrective Diffusion Language Models},
    author={Zhang, Shuibai and Peng, Fred Zhangzhi and Zhang, Yiheng and Pan, Jin and Chrysos, Grigorios G},
    journal={arXiv preprint arXiv:2512.15596},
    year={2025}
}

@inproceedings{rutte-etal:2025generalized,
    title={Generalized Interpolating Discrete Diffusion},
    author={R{\"u}tte, Dimitri von and Janis Fluri and Yuhui Ding and Antonio Orvieto and Bernhard Sch{\"o}lkopf and Thomas Hofmann},
    booktitle={Forty-second International Conference on Machine Learning},
    year={2025}
}

@article{gu-etal:2019levenshtein,
    title={Levenshtein transformer},
    author={Gu, Jiatao and Wang, Changhan and Zhao, Junbo},
    journal={Advances in neural information processing systems},
    volume={32},
    year={2019}
}

@article{reid-etal:2022diffuser,
    title={Diffuser: Discrete diffusion via edit-based reconstruction},
    author={Reid, Machel and Hellendoorn, Vincent J and Neubig, Graham},
    journal={arXiv preprint arXiv:2210.16886},
    year={2022}
}

@inproceedings{tay-etal:2023ul2,
    title={{UL}2: Unifying Language Learning Paradigms},
    author={Yi Tay and Mostafa Dehghani and Vinh Q. Tran and Xavier Garcia and Jason Wei and Xuezhi Wang and Hyung Won Chung and Dara Bahri and Tal Schuster and Steven Zheng and Denny Zhou and Neil Houlsby and Donald Metzler},
    booktitle={The Eleventh International Conference on Learning Representations },
    year={2023}
}

@inproceedings{zhu-etal:2020freelb,
    title={FreeLB: Enhanced Adversarial Training for Natural Language Understanding},
    author={Chen Zhu and Yu Cheng and Zhe Gan and Siqi Sun and Tom Goldstein and Jingjing Liu},
    booktitle={International Conference on Learning Representations},
    year={2020}
}

@inproceedings{jang-etal:2017categorical,
    title={Categorical Reparameterization with Gumbel-Softmax},
    author={Eric Jang and Shixiang Gu and Ben Poole},
    booktitle={International Conference on Learning Representations},
    year={2017}
}

@inproceedings{maddison-etal:2017the,
    title={The Concrete Distribution: A Continuous Relaxation of Discrete Random Variables},
    author={Chris J. Maddison and Andriy Mnih and Yee Whye Teh},
    booktitle={International Conference on Learning Representations},
    year={2017}
}

@article{hoogeboom-etal:2021argmax,
    title={Argmax flows and multinomial diffusion: Learning categorical distributions},
    author={Hoogeboom, Emiel and Nielsen, Didrik and Jaini, Priyank and Forr{\'e}, Patrick and Welling, Max},
    journal={Advances in neural information processing systems},
    volume={34},
    pages={12454--12465},
    year={2021}
}

@inproceedings{lou-etal:2024discrete,
    title={Discrete diffusion modeling by estimating the ratios of the data distribution},
    author={Lou, Aaron and Meng, Chenlin and Ermon, Stefano},
    booktitle={Proceedings of the 41st International Conference on Machine Learning},
    pages={32819--32848},
    year={2024}
}

@inproceedings{lee-etal:2018deterministic,
    title={Deterministic Non-Autoregressive Neural Sequence Modeling by Iterative Refinement},
    author={Lee, Jason and Mansimov, Elman and Cho, Kyunghyun},
    booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
    pages={1173--1182},
    year={2018}
}

@inproceedings{koh-etal:2025conditional,
    title={Conditional [MASK] Discrete Diffusion Language Model},
    author={Koh, Hyukhun and Jhang, Minha and Kim, Dohyung and Lee, Sangmook and Jung, Kyomin},
    booktitle={Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing},
    pages={8910--8934},
    year={2025}
}

@article{gat-etal:2024discrete,
    title={Discrete flow matching},
    author={Gat, Itai and Remez, Tal and Shaul, Neta and Kreuk, Felix and Chen, Ricky TQ and Synnaeve, Gabriel and Adi, Yossi and Lipman, Yaron},
    journal={Advances in Neural Information Processing Systems},
    volume={37},
    pages={133345--133385},
    year={2024}
}

@article{monsefi-etal:2025fs,
    title={Fs-dfm: Fast and accurate long text generation with few-step diffusion language models},
    author={Monsefi, Amin Karimi and Bhendawade, Nikhil and Ciosici, Manuel Rafael and Culver, Dominic and Zhang, Yizhe and Belousova, Irina},
    journal={arXiv preprint arXiv:2509.20624},
    year={2025}
}

@inproceedings{peebles-and-xie:2023scalable,
    title={Scalable diffusion models with transformers},
    author={Peebles, William and Xie, Saining},
    booktitle={Proceedings of the IEEE/CVF international conference on computer vision},
    pages={4195--4205},
    year={2023}
}

@article{ho-eatl:2020denoising,
    title={Denoising diffusion probabilistic models},
    author={Ho, Jonathan and Jain, Ajay and Abbeel, Pieter},
    journal={Advances in neural information processing systems},
    volume={33},
    pages={6840--6851},
    year={2020}
}

@inproceedings{salimans-and-ho:2022progressive,
    title={Progressive Distillation for Fast Sampling of Diffusion Models},
    author={Tim Salimans and Jonathan Ho},
    booktitle={International Conference on Learning Representations},
    year={2022}
}

@inproceedings{campbell-etal:2024generative,
    title={Generative Flows on Discrete State-Spaces: Enabling Multimodal Flows with Applications to Protein Co-Design},
    author={Campbell, Andrew and Yim, Jason and Barzilay, Regina and Rainforth, Tom and Jaakkola, Tommi},
    booktitle={International Conference on Machine Learning},
    pages={5453--5512},
    year={2024},
    organization={PMLR}
}

@inproceedings{shaul-etal:2025flow,
    title={Flow Matching with General Discrete Paths: A Kinetic-Optimal Perspective},
    author={Neta Shaul and Itai Gat and Marton Havasi and Daniel Severo and Anuroop Sriram and Peter Holderrieth and Brian Karrer and Yaron Lipman and Ricky T. Q. Chen},
    booktitle={The Thirteenth International Conference on Learning Representations},
    year={2025}
}

@article{gillespie:1977exact,
    title={Exact stochastic simulation of coupled chemical reactions},
    author={Gillespie, Daniel T},
    journal={The journal of physical chemistry},
    volume={81},
    number={25},
    pages={2340--2361},
    year={1977},
    publisher={ACS Publications}
}

@article{gillespie:2001approximate,
    title={Approximate accelerated stochastic simulation of chemically reacting systems},
    author={Gillespie, Daniel T},
    journal={The Journal of chemical physics},
    volume={115},
    number={4},
    pages={1716--1733},
    year={2001},
    publisher={AIP Publishing}
}

@article{lecun-etal:2006tutorial,
   title={A tutorial on energy-based learning},
    author={LeCun, Yann and Chopra, Sumit and Hadsell, Raia and Ranzato, M and Huang, Fujie and others},
    journal={Predicting structured data},
    volume={1},
    year={2006}
}

@article{lipman-etal:2024flow,
    title={Flow matching guide and code},
    author={Lipman, Yaron and Havasi, Marton and Holderrieth, Peter and Shaul, Neta and Le, Matt and Karrer, Brian and Chen, Ricky TQ and Lopez-Paz, David and Ben-Hamu, Heli and Gat, Itai},
    journal={arXiv preprint arXiv:2412.06264},
    year={2024}
}

@inproceedings{zheng-etal:2023dpmsolverv,
title={{DPM}-Solver-v3: Improved Diffusion {ODE} Solver with Empirical Model Statistics},
author={Kaiwen Zheng and Cheng Lu and Jianfei Chen and Jun Zhu},
booktitle={Thirty-seventh Conference on Neural Information Processing Systems},
year={2023},
url={https://openreview.net/forum?id=9fWKExmKa0}
}

@article{Guo-etal:2025DeepSeekR1,
  author       = {Daya Guo and
                  Dejian Yang and
                  Haowei Zhang and
                  Junxiao Song and
                  Peiyi Wang and
                  Qihao Zhu and
                  Runxin Xu and
                  Ruoyu Zhang and
                  Shirong Ma and
                  Xiao Bi and
                  Xiaokang Zhang and
                  Xingkai Yu and
                  Yu Wu and
                  Z. F. Wu and
                  Zhibin Gou and
                  Zhihong Shao and
                  Zhuoshu Li and
                  Ziyi Gao and
                  Aixin Liu and
                  Bing Xue and
                  Bingxuan Wang and
                  Bochao Wu and
                  Bei Feng and
                  Chengda Lu and
                  Chenggang Zhao and
                  Chengqi Deng and
                  Chong Ruan and
                  Damai Dai and
                  Deli Chen and
                  Dongjie Ji and
                  Erhang Li and
                  Fangyun Lin and
                  Fucong Dai and
                  Fuli Luo and
                  Guangbo Hao and
                  Guanting Chen and
                  Guowei Li and
                  Hao Zhang and
                  Hanwei Xu and
                  Honghui Ding and
                  Huazuo Gao and
                  Hui Qu and
                  Hui Li and
                  Jianzhong Guo and
                  Jiashi Li and
                  Jingchang Chen and
                  Jingyang Yuan and
                  Jinhao Tu and
                  Junjie Qiu and
                  Junlong Li and
                  J. L. Cai and
                  Jiaqi Ni and
                  Jian Liang and
                  Jin Chen and
                  Kai Dong and
                  Kai Hu and
                  Kaichao You and
                  Kaige Gao and
                  Kang Guan and
                  Kexin Huang and
                  Kuai Yu and
                  Lean Wang and
                  Lecong Zhang and
                  Liang Zhao and
                  Litong Wang and
                  Liyue Zhang and
                  Lei Xu and
                  Leyi Xia and
                  Mingchuan Zhang and
                  Minghua Zhang and
                  Minghui Tang and
                  Mingxu Zhou and
                  Meng Li and
                  Miaojun Wang and
                  Mingming Li and
                  Ning Tian and
                  Panpan Huang and
                  Peng Zhang and
                  Qiancheng Wang and
                  Qinyu Chen and
                  Qiushi Du and
                  Ruiqi Ge and
                  Ruisong Zhang and
                  Ruizhe Pan and
                  Runji Wang and
                  R. J. Chen and
                  R. L. Jin and
                  Ruyi Chen and
                  Shanghao Lu and
                  Shangyan Zhou and
                  Shanhuang Chen and
                  Shengfeng Ye and
                  Shiyu Wang and
                  Shuiping Yu and
                  Shunfeng Zhou and
                  Shuting Pan and
                  S. S. Li and
                  Shuang Zhou and
                  Shaoqing Wu and
                  Tao Yun and
                  Tian Pei and
                  Tianyu Sun and
                  Tao Wang and
                  Wangding Zeng and
                  Wen Liu and
                  Wenfeng Liang and
                  Wenjun Gao and
                  Wenqin Yu and
                  Wentao Zhang and
                  W. L. Xiao and
                  Wei An and
                  Xiaodong Liu and
                  Xiaohan Wang and
                  Xiaokang Chen and
                  Xiaotao Nie and
                  Xin Cheng and
                  Xin Liu and
                  Xin Xie and
                  Xingchao Liu and
                  Xinyu Yang and
                  Xinyuan Li and
                  Xuecheng Su and
                  Xuheng Lin and
                  X. Q. Li and
                  Xiangyue Jin and
                  Xiaojin Shen and
                  Xiaosha Chen and
                  Xiaowen Sun and
                  Xiaoxiang Wang and
                  Xinnan Song and
                  Xinyi Zhou and
                  Xianzu Wang and
                  Xinxia Shan and
                  Y. K. Li and
                  Y. Q. Wang and
                  Y. X. Wei and
                  Yang Zhang and
                  Yanhong Xu and
                  Yao Li and
                  Yao Zhao and
                  Yaofeng Sun and
                  Yaohui Wang and
                  Yi Yu and
                  Yichao Zhang and
                  Yifan Shi and
                  Yiliang Xiong and
                  Ying He and
                  Yishi Piao and
                  Yisong Wang and
                  Yixuan Tan and
                  Yiyang Ma and
                  Yiyuan Liu and
                  Yongqiang Guo and
                  Yuan Ou and
                  Yuduan Wang and
                  Yue Gong and
                  Yuheng Zou and
                  Yujia He and
                  Yunfan Xiong and
                  Yuxiang Luo and
                  Yuxiang You and
                  Yuxuan Liu and
                  Yuyang Zhou and
                  Y. X. Zhu and
                  Yanping Huang and
                  Yaohui Li and
                  Yi Zheng and
                  Yuchen Zhu and
                  Yunxian Ma and
                  Ying Tang and
                  Yukun Zha and
                  Yuting Yan and
                  Z. Z. Ren and
                  Zehui Ren and
                  Zhangli Sha and
                  Zhe Fu and
                  Zhean Xu and
                  Zhenda Xie and
                  Zhengyan Zhang and
                  Zhewen Hao and
                  Zhicheng Ma and
                  Zhigang Yan and
                  Zhiyu Wu and
                  Zihui Gu and
                  Zijia Zhu and
                  Zijun Liu and
                  Zilin Li and
                  Ziwei Xie and
                  Ziyang Song and
                  Zizheng Pan and
                  Zhen Huang and
                  Zhipeng Xu and
                  Zhongyu Zhang and
                  Zhen Zhang},
  title        = {DeepSeek-R1 incentivizes reasoning in LLMs through reinforcement learning},
  journal      = {Nat.},
  volume       = {645},
  number       = {8081},
  pages        = {633--638},
  year         = {2025},
  url          = {https://doi.org/10.1038/s41586-025-09422-z},
  doi          = {10.1038/S41586-025-09422-Z},
  timestamp    = {Sun, 02 Nov 2025 21:29:27 +0100},
  biburl       = {https://dblp.org/rec/journals/nature/GuoYZSWZXZMBZY025.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{yang-etal:2024loopedTransformer,
title={Looped Transformers are Better at Learning Learning Algorithms},
author={Liu Yang and Kangwook Lee and Robert D Nowak and Dimitris Papailiopoulos},
booktitle={The Twelfth International Conference on Learning Representations},
year={2024},
url={https://openreview.net/forum?id=HHbRxoDTxE}
}

@misc{alexnet,
  note = {Krizhevsky A, Sutskever I, Hinton G E. ImageNet Classification with Deep Convolutional Neural Networks. NeurIPS, 2012.}
}

@misc{vgg,
  note = {Simonyan K, Zisserman A. Very Deep Convolutional Networks for Large-Scale Image Recognition (VGG). ICLR, 2015.}
}

@misc{resnet,
  note = {He K, Zhang X, Ren S, Sun J. Deep Residual Learning for Image Recognition. CVPR, 2016.}
}

@misc{vit,
  note = {Dosovitskiy A, et al. An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale. ICLR, 2021.}
}

@misc{simclr,
  note = {Chen T, et al. A Simple Framework for Contrastive Learning of Visual Representations (SimCLR). ICML, 2020.}
}

@misc{moco,
  note = {He K, et al. Momentum Contrast for Unsupervised Visual Representation Learning (MoCo). CVPR, 2020.}
}

@misc{dino,
  note = {Caron M, et al. Emerging Properties in Self-Supervised Vision Transformers (DINO). ICCV, 2021.}
}

@misc{cpc,
  note = {van den Oord A, Li Y, Vinyals O. Representation Learning with Contrastive Predictive Coding (CPC). arXiv:1807.03748, 2018.}
}

@misc{wav2vec2,
  note = {Baevski A, Zhou H, Mohamed A, Auli M. wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations. NeurIPS, 2020.}
}

@misc{hubert,
  note = {Hsu W N, et al. HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units. IEEE/ACM Transactions on Audio, Speech, and Language Processing, 2021.}
}

@misc{data2vec,
  note = {Baevski A, et al. data2vec: A General Framework for Self-Supervised Learning in Speech, Vision and Language. ICML, 2022.}
}

@misc{mae,
  note = {He K, et al. Masked Autoencoders Are Scalable Vision Learners (MAE). CVPR, 2022.}
}

@misc{beit,
  note = {Bao H, Dong L, Piao S, Wei F. BEiT: BERT Pre-Training of Image Transformers. ICLR, 2022.}
}

@misc{clip,
  note = {Radford A, et al. Learning Transferable Visual Models From Natural Language Supervision (CLIP). ICML, 2021.}
}

@misc{align,
  note = {Jia C, et al. Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision (ALIGN). ICML, 2021.}
}

@misc{siglip,
  note = {Zhai X, et al. Sigmoid Loss for Language Image Pre-Training (SigLIP). ICCV, 2023.}
}

@misc{waco,
  note = {Ouyang S, Ye R, Li L. WACO: Word-Aligned Contrastive Learning for Speech Translation. ACL, 2023.}
}

@misc{clap,
  note = {Elizalde B, Deshmukh S, Al Ismail M, Wang H. CLAP: Learning Audio Concepts From Natural Language Supervision. ICASSP, 2023.}
}

@misc{flamingo,
  note = {Alayrac J B, et al. Flamingo: a Visual Language Model for Few-Shot Learning. NeurIPS, 2022.}
}

@misc{blip2,
  note = {Li J, et al. BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models. ICML, 2023.}
}

@misc{llava,
  note = {Liu H, et al. Visual Instruction Tuning (LLaVA). NeurIPS, 2023.}
}

@misc{instructblip,
  note = {Dai W, et al. InstructBLIP: Towards General-purpose Vision-Language Models with Instruction Tuning. NeurIPS, 2023.}
}

@misc{qwenvl,
  note = {Bai J, et al. Qwen-VL: A Versatile Vision-Language Model for Understanding, Localization, Text Reading, and Beyond. arXiv:2308.12966, 2023.}
}

@misc{whisper,
  note = {Radford A, et al. Robust Speech Recognition via Large-Scale Weak Supervision (Whisper). ICML, 2023.}
}

@misc{speechgpt,
  note = {Zhang D, et al. SpeechGPT: Empowering Large Language Models with Intrinsic Cross-Modal Conversational Abilities. Findings of EMNLP, 2023.}
}

@misc{qwenaudio,
  note = {Chu Y, et al. Qwen-Audio: Advancing Universal Audio Understanding via Unified Large-Scale Audio-Language Models. arXiv:2311.07919, 2023.}
}

@misc{salmonn,
  note = {Tang C, et al. SALMONN: Towards Generic Hearing Abilities for Large Language Models. ICLR, 2024.}
}

@misc{emu,
  note = {Sun Q, et al. Emu: Generative Pretraining in Multimodality. ICLR, 2024.}
}

@misc{gan,
  note = {Goodfellow I, et al. Generative Adversarial Nets. NeurIPS, 2014.}
}

@misc{vae,
  note = {Kingma D P, Welling M. Auto-Encoding Variational Bayes (VAE). ICLR, 2014.}
}

@misc{ddpm,
  note = {Ho J, Jain A, Abbeel P. Denoising Diffusion Probabilistic Models (DDPM). NeurIPS, 2020.}
}

@misc{cfg,
  note = {Ho J, Salimans T. Classifier-Free Diffusion Guidance. arXiv:2207.12598, 2022.}
}

@misc{dalle,
  note = {Ramesh A, et al. Zero-Shot Text-to-Image Generation (DALL·E). ICML, 2021.}
}

@misc{glide,
  note = {Nichol A, et al. GLIDE: Towards Photorealistic Image Generation and Editing with Text-Guided Diffusion Models. ICML, 2022.}
}

@misc{dalle2,
  note = {Ramesh A, et al. Hierarchical Text-Conditional Image Generation with CLIP Latents (DALL·E 2). arXiv:2204.06125, 2022.}
}

@misc{imagen,
  note = {Saharia C, et al. Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding (Imagen). NeurIPS, 2022.}
}

@misc{ldm,
  note = {Rombach R, et al. High-Resolution Image Synthesis with Latent Diffusion Models (Stable Diffusion). CVPR, 2022.}
}

@misc{sdedit,
  note = {Meng C, et al. SDEdit: Guided Image Synthesis and Editing with Stochastic Differential Equations. ICLR, 2022.}
}

@misc{p2p,
  note = {Hertz A, et al. Prompt-to-Prompt Image Editing with Cross Attention Control. ICLR, 2023.}
}

@misc{ip2p,
  note = {Brooks T, Holynski A, Efros A A. InstructPix2Pix: Learning to Follow Image Editing Instructions. CVPR, 2023.}
}

@misc{controlnet,
  note = {Zhang L, Rao A, Agrawala M. Adding Conditional Control to Text-to-Image Diffusion Models (ControlNet). ICCV, 2023.}
}

@misc{tacotron2,
  note = {Shen J, et al. Natural TTS Synthesis by Conditioning WaveNet on Mel Spectrogram Predictions (Tacotron 2). ICASSP, 2018.}
}

@misc{fastspeech2,
  note = {Ren Y, et al. FastSpeech 2: Fast and High-Quality End-to-End Text to Speech. ICLR, 2021.}
}

@misc{vits,
  note = {Kim J, Kong J, Son J. Conditional Variational Autoencoder with Adversarial Learning for End-to-End Text-to-Speech (VITS). ICML, 2021.}
}

@misc{valle,
  note = {Wang C, et al. Neural Codec Language Models are Zero-Shot Text to Speech Synthesizers (VALL-E). arXiv:2301.02111, 2023.}
}

@misc{nist-synthetic,
  note = {National Institute of Standards and Technology. Reducing Risks Posed by Synthetic Content. NIST AI 100-4, 2024.}
}

@misc{dreambooth,
  note = {Ruiz N, et al. DreamBooth: Fine Tuning Text-to-Image Diffusion Models for Subject-Driven Generation. CVPR, 2023.}
}

@misc{audiogen,
  note = {Kreuk F, et al. AudioGen: Textually Guided Audio Generation. ICLR, 2023.}
}

@misc{audioldm,
  note = {Liu H, et al. AudioLDM: Text-to-Audio Generation with Latent Diffusion Models. ICML, 2023.}
}

@misc{makeanaudio,
  note = {Huang R, et al. Make-An-Audio: Text-To-Audio Generation with Prompt-Enhanced Diffusion Models. ICML, 2023.}
}

@misc{makeanaudio2,
  note = {Huang J, et al. Make-An-Audio 2: Temporal-Enhanced Text-to-Audio Generation. arXiv:2305.18474, 2023.}
}

@misc{audit,
  note = {Wang Y, et al. AUDIT: Audio Editing by Following Instructions with Latent Diffusion Models. NeurIPS, 2023.}
}

@misc{codi,
  note = {Tang Z, et al. Any-to-Any Generation via Composable Diffusion (CoDi). NeurIPS, 2023.}
}

@misc{yao-etal:2023tree,
  title = {Tree of Thoughts},
  note = {Placeholder entry added during textbook merge; replace with full bibliographic metadata.},
  year = {2023}
}

@misc{zheng-etal:2023judging,
  title = {Judging large language model reasoning with generative verification},
  note = {Placeholder entry added during textbook merge; replace with full bibliographic metadata.},
  year = {2023}
}

@misc{wang-etal:2024math,
  title = {Math reasoning verification with rollout},
  note = {Placeholder entry added during textbook merge; replace with full bibliographic metadata.},
  year = {2024}
}
