Links to Exploration step
Le document en format XML
<record><TEI><teiHeader><fileDesc><titleStmt><title xml:lang="en">An expanded evaluation of protein function prediction methods shows an improvement in accuracy</title><author><name sortKey="Jiang, Yuxiang" sort="Jiang, Yuxiang" uniqKey="Jiang Y" first="Yuxiang" last="Jiang">Yuxiang Jiang</name><affiliation><nlm:aff id="Aff1">Department of Computer Science and Informatics, Indiana University, Bloomington, IN USA</nlm:aff></affiliation></author><author><name sortKey="Oron, Tal Ronnen" sort="Oron, Tal Ronnen" uniqKey="Oron T" first="Tal Ronnen" last="Oron">Tal Ronnen Oron</name><affiliation><nlm:aff id="Aff2">Buck Institute for Research on Aging, Novato, CA USA</nlm:aff></affiliation></author><author><name sortKey="Clark, Wyatt T" sort="Clark, Wyatt T" uniqKey="Clark W" first="Wyatt T." last="Clark">Wyatt T. Clark</name><affiliation><nlm:aff id="Aff3">Department of Molecular Biophysics and Biochemistry, Yale University, New Haven, CT USA</nlm:aff></affiliation></author><author><name sortKey="Bankapur, Asma R" sort="Bankapur, Asma R" uniqKey="Bankapur A" first="Asma R." last="Bankapur">Asma R. Bankapur</name><affiliation><nlm:aff id="Aff4">Department of Microbiology, Miami University, Oxford, OH USA</nlm:aff></affiliation></author><author><name sortKey="D Ndrea, Daniel" sort="D Ndrea, Daniel" uniqKey="D Ndrea D" first="Daniel" last="D Ndrea">Daniel D Ndrea</name><affiliation><nlm:aff id="Aff5">University of Rome, La Sapienza, Rome, Italy</nlm:aff></affiliation></author><author><name sortKey="Lepore, Rosalba" sort="Lepore, Rosalba" uniqKey="Lepore R" first="Rosalba" last="Lepore">Rosalba Lepore</name><affiliation><nlm:aff id="Aff5">University of Rome, La Sapienza, Rome, Italy</nlm:aff></affiliation></author><author><name sortKey="Funk, Christopher S" sort="Funk, Christopher S" uniqKey="Funk C" first="Christopher S." last="Funk">Christopher S. Funk</name><affiliation><nlm:aff id="Aff6">Computational Bioscience Program, University of Colorado School of Medicine, Aurora, CO USA</nlm:aff></affiliation></author><author><name sortKey="Kahanda, Indika" sort="Kahanda, Indika" uniqKey="Kahanda I" first="Indika" last="Kahanda">Indika Kahanda</name><affiliation><nlm:aff id="Aff7">Department of Computer Science, Colorado State University, Fort Collins, CO USA</nlm:aff></affiliation></author><author><name sortKey="Verspoor, Karin M" sort="Verspoor, Karin M" uniqKey="Verspoor K" first="Karin M." last="Verspoor">Karin M. Verspoor</name><affiliation><nlm:aff id="Aff8">Department of Computing and Information Systems, University of Melbourne, Parkville, Victoria, Australia</nlm:aff></affiliation><affiliation><nlm:aff id="Aff9">Health and Biomedical Informatics Centre, University of Melbourne, Parkville, Victoria, Australia</nlm:aff></affiliation></author><author><name sortKey="Ben Hur, Asa" sort="Ben Hur, Asa" uniqKey="Ben Hur A" first="Asa" last="Ben-Hur">Asa Ben-Hur</name><affiliation><nlm:aff id="Aff7">Department of Computer Science, Colorado State University, Fort Collins, CO USA</nlm:aff></affiliation></author><author><name sortKey="Koo, Da Chen Emily" sort="Koo, Da Chen Emily" uniqKey="Koo D" first="Da Chen Emily" last="Koo">Da Chen Emily Koo</name><affiliation><nlm:aff id="Aff10">Department of Biology, New York University, New York, NY USA</nlm:aff></affiliation></author><author><name sortKey="Penfold Brown, Duncan" sort="Penfold Brown, Duncan" uniqKey="Penfold Brown D" first="Duncan" last="Penfold-Brown">Duncan Penfold-Brown</name><affiliation><nlm:aff id="Aff11">Social Media and Political Participation Lab, New York University, New York, NY USA</nlm:aff></affiliation><affiliation><nlm:aff id="Aff12">CY Data Science, New York, NY USA</nlm:aff></affiliation></author><author><name sortKey="Shasha, Dennis" sort="Shasha, Dennis" uniqKey="Shasha D" first="Dennis" last="Shasha">Dennis Shasha</name><affiliation><nlm:aff id="Aff13">Department of Computer Science, New York University, New York, NY USA</nlm:aff></affiliation></author><author><name sortKey="Youngs, Noah" sort="Youngs, Noah" uniqKey="Youngs N" first="Noah" last="Youngs">Noah Youngs</name><affiliation><nlm:aff id="Aff12">CY Data Science, New York, NY USA</nlm:aff></affiliation><affiliation><nlm:aff id="Aff13">Department of Computer Science, New York University, New York, NY USA</nlm:aff></affiliation><affiliation><nlm:aff id="Aff14">Simons Center for Data Analysis, New York, NY USA</nlm:aff></affiliation></author><author><name sortKey="Bonneau, Richard" sort="Bonneau, Richard" uniqKey="Bonneau R" first="Richard" last="Bonneau">Richard Bonneau</name><affiliation><nlm:aff id="Aff13">Department of Computer Science, New York University, New York, NY USA</nlm:aff></affiliation><affiliation><nlm:aff id="Aff14">Simons Center for Data Analysis, New York, NY USA</nlm:aff></affiliation><affiliation><nlm:aff id="Aff15">Center for Genomics and Systems Biology, Department of Biology, New York University, New York, NY USA</nlm:aff></affiliation></author><author><name sortKey="Lin, Alexandra" sort="Lin, Alexandra" uniqKey="Lin A" first="Alexandra" last="Lin">Alexandra Lin</name><affiliation><nlm:aff id="Aff16">Department of Electrical Engineering and Computer Sciences, University of California Berkeley, Berkeley, CA USA</nlm:aff></affiliation></author><author><name sortKey="Sahraeian, Sayed M E" sort="Sahraeian, Sayed M E" uniqKey="Sahraeian S" first="Sayed M. E." last="Sahraeian">Sayed M. E. Sahraeian</name><affiliation><nlm:aff id="Aff17">Department of Plant and Microbial Biology, University of California Berkeley, Berkeley, CA USA</nlm:aff></affiliation></author><author><name sortKey="Martelli, Pier Luigi" sort="Martelli, Pier Luigi" uniqKey="Martelli P" first="Pier Luigi" last="Martelli">Pier Luigi Martelli</name><affiliation><nlm:aff id="Aff18">Biocomputing Group, BiGeA, University of Bologna, Bologna, Italy</nlm:aff></affiliation></author><author><name sortKey="Profiti, Giuseppe" sort="Profiti, Giuseppe" uniqKey="Profiti G" first="Giuseppe" last="Profiti">Giuseppe Profiti</name><affiliation><nlm:aff id="Aff18">Biocomputing Group, BiGeA, University of Bologna, Bologna, Italy</nlm:aff></affiliation></author><author><name sortKey="Casadio, Rita" sort="Casadio, Rita" uniqKey="Casadio R" first="Rita" last="Casadio">Rita Casadio</name><affiliation><nlm:aff id="Aff18">Biocomputing Group, BiGeA, University of Bologna, Bologna, Italy</nlm:aff></affiliation></author><author><name sortKey="Cao, Renzhi" sort="Cao, Renzhi" uniqKey="Cao R" first="Renzhi" last="Cao">Renzhi Cao</name><affiliation><nlm:aff id="Aff19">Computer Science Department, University of Missouri, Columbia, MO USA</nlm:aff></affiliation></author><author><name sortKey="Zhong, Zhaolong" sort="Zhong, Zhaolong" uniqKey="Zhong Z" first="Zhaolong" last="Zhong">Zhaolong Zhong</name><affiliation><nlm:aff id="Aff19">Computer Science Department, University of Missouri, Columbia, MO USA</nlm:aff></affiliation></author><author><name sortKey="Cheng, Jianlin" sort="Cheng, Jianlin" uniqKey="Cheng J" first="Jianlin" last="Cheng">Jianlin Cheng</name><affiliation><nlm:aff id="Aff19">Computer Science Department, University of Missouri, Columbia, MO USA</nlm:aff></affiliation></author><author><name sortKey="Altenhoff, Adrian" sort="Altenhoff, Adrian" uniqKey="Altenhoff A" first="Adrian" last="Altenhoff">Adrian Altenhoff</name><affiliation><nlm:aff id="Aff20">ETH Zurich, Zurich, Switzerland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff21">Swiss Institute of Bioinformatics, Zurich, Switzerland</nlm:aff></affiliation></author><author><name sortKey="Skunca, Nives" sort="Skunca, Nives" uniqKey="Skunca N" first="Nives" last="Skunca">Nives Skunca</name><affiliation><nlm:aff id="Aff20">ETH Zurich, Zurich, Switzerland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff21">Swiss Institute of Bioinformatics, Zurich, Switzerland</nlm:aff></affiliation></author><author><name sortKey="Dessimoz, Christophe" sort="Dessimoz, Christophe" uniqKey="Dessimoz C" first="Christophe" last="Dessimoz">Christophe Dessimoz</name><affiliation><nlm:aff id="Aff22">Bioinformatics Group, Department of Computer Science, University College London, London, UK</nlm:aff></affiliation><affiliation><nlm:aff id="Aff87">University of Lausanne, Lausanne, Switzerland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff88">Swiss Institute of Bioinformatics, Lausanne, Switzerland</nlm:aff></affiliation></author><author><name sortKey="Dogan, Tunca" sort="Dogan, Tunca" uniqKey="Dogan T" first="Tunca" last="Dogan">Tunca Dogan</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Hakala, Kai" sort="Hakala, Kai" uniqKey="Hakala K" first="Kai" last="Hakala">Kai Hakala</name><affiliation><nlm:aff id="Aff24">Department of Information Technology, University of Turku, Turku, Finland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff25">University of Turku Graduate School, University of Turku, Turku, Finland</nlm:aff></affiliation></author><author><name sortKey="Kaewphan, Suwisa" sort="Kaewphan, Suwisa" uniqKey="Kaewphan S" first="Suwisa" last="Kaewphan">Suwisa Kaewphan</name><affiliation><nlm:aff id="Aff24">Department of Information Technology, University of Turku, Turku, Finland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff25">University of Turku Graduate School, University of Turku, Turku, Finland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff26">Turku Centre for Computer Science, Turku, Finland</nlm:aff></affiliation></author><author><name sortKey="Mehryary, Farrokh" sort="Mehryary, Farrokh" uniqKey="Mehryary F" first="Farrokh" last="Mehryary">Farrokh Mehryary</name><affiliation><nlm:aff id="Aff24">Department of Information Technology, University of Turku, Turku, Finland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff25">University of Turku Graduate School, University of Turku, Turku, Finland</nlm:aff></affiliation></author><author><name sortKey="Salakoski, Tapio" sort="Salakoski, Tapio" uniqKey="Salakoski T" first="Tapio" last="Salakoski">Tapio Salakoski</name><affiliation><nlm:aff id="Aff24">Department of Information Technology, University of Turku, Turku, Finland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff26">Turku Centre for Computer Science, Turku, Finland</nlm:aff></affiliation></author><author><name sortKey="Ginter, Filip" sort="Ginter, Filip" uniqKey="Ginter F" first="Filip" last="Ginter">Filip Ginter</name><affiliation><nlm:aff id="Aff24">Department of Information Technology, University of Turku, Turku, Finland</nlm:aff></affiliation></author><author><name sortKey="Fang, Hai" sort="Fang, Hai" uniqKey="Fang H" first="Hai" last="Fang">Hai Fang</name><affiliation><nlm:aff id="Aff27">University of Bristol, Bristol, UK</nlm:aff></affiliation></author><author><name sortKey="Smithers, Ben" sort="Smithers, Ben" uniqKey="Smithers B" first="Ben" last="Smithers">Ben Smithers</name><affiliation><nlm:aff id="Aff27">University of Bristol, Bristol, UK</nlm:aff></affiliation></author><author><name sortKey="Oates, Matt" sort="Oates, Matt" uniqKey="Oates M" first="Matt" last="Oates">Matt Oates</name><affiliation><nlm:aff id="Aff27">University of Bristol, Bristol, UK</nlm:aff></affiliation></author><author><name sortKey="Gough, Julian" sort="Gough, Julian" uniqKey="Gough J" first="Julian" last="Gough">Julian Gough</name><affiliation><nlm:aff id="Aff27">University of Bristol, Bristol, UK</nlm:aff></affiliation></author><author><name sortKey="Toronen, Petri" sort="Toronen, Petri" uniqKey="Toronen P" first="Petri" last="Törönen">Petri Törönen</name><affiliation><nlm:aff id="Aff28">Institute of Biotechnology, University of Helsinki, Helsinki, Finland</nlm:aff></affiliation></author><author><name sortKey="Koskinen, Patrik" sort="Koskinen, Patrik" uniqKey="Koskinen P" first="Patrik" last="Koskinen">Patrik Koskinen</name><affiliation><nlm:aff id="Aff28">Institute of Biotechnology, University of Helsinki, Helsinki, Finland</nlm:aff></affiliation></author><author><name sortKey="Holm, Liisa" sort="Holm, Liisa" uniqKey="Holm L" first="Liisa" last="Holm">Liisa Holm</name><affiliation><nlm:aff id="Aff28">Institute of Biotechnology, University of Helsinki, Helsinki, Finland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff86">Department of Biological and Environmental Sciences, Universitity of Helsinki, Helsinki, Finland</nlm:aff></affiliation></author><author><name sortKey="Chen, Ching Tai" sort="Chen, Ching Tai" uniqKey="Chen C" first="Ching-Tai" last="Chen">Ching-Tai Chen</name><affiliation><nlm:aff id="Aff29">Institute of Information Science, Academia Sinica, Taipei, Taiwan</nlm:aff></affiliation></author><author><name sortKey="Hsu, Wen Lian" sort="Hsu, Wen Lian" uniqKey="Hsu W" first="Wen-Lian" last="Hsu">Wen-Lian Hsu</name><affiliation><nlm:aff id="Aff29">Institute of Information Science, Academia Sinica, Taipei, Taiwan</nlm:aff></affiliation></author><author><name sortKey="Bryson, Kevin" sort="Bryson, Kevin" uniqKey="Bryson K" first="Kevin" last="Bryson">Kevin Bryson</name><affiliation><nlm:aff id="Aff22">Bioinformatics Group, Department of Computer Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Cozzetto, Domenico" sort="Cozzetto, Domenico" uniqKey="Cozzetto D" first="Domenico" last="Cozzetto">Domenico Cozzetto</name><affiliation><nlm:aff id="Aff22">Bioinformatics Group, Department of Computer Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Minneci, Federico" sort="Minneci, Federico" uniqKey="Minneci F" first="Federico" last="Minneci">Federico Minneci</name><affiliation><nlm:aff id="Aff22">Bioinformatics Group, Department of Computer Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Jones, David T" sort="Jones, David T" uniqKey="Jones D" first="David T." last="Jones">David T. Jones</name><affiliation><nlm:aff id="Aff22">Bioinformatics Group, Department of Computer Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Chapman, Samuel" sort="Chapman, Samuel" uniqKey="Chapman S" first="Samuel" last="Chapman">Samuel Chapman</name><affiliation><nlm:aff id="Aff30">Department of Computational Science and Engineering, North Carolina A&T State University, Greensboro, NC USA</nlm:aff></affiliation></author><author><name sortKey="Bkc, Dukka" sort="Bkc, Dukka" uniqKey="Bkc D" first="Dukka" last="Bkc">Dukka Bkc</name><affiliation><nlm:aff id="Aff30">Department of Computational Science and Engineering, North Carolina A&T State University, Greensboro, NC USA</nlm:aff></affiliation></author><author><name sortKey="Khan, Ishita K" sort="Khan, Ishita K" uniqKey="Khan I" first="Ishita K." last="Khan">Ishita K. Khan</name><affiliation><nlm:aff id="Aff31">Department of Computer Science, Purdue University, West Lafayette, IN USA</nlm:aff></affiliation></author><author><name sortKey="Kihara, Daisuke" sort="Kihara, Daisuke" uniqKey="Kihara D" first="Daisuke" last="Kihara">Daisuke Kihara</name><affiliation><nlm:aff id="Aff31">Department of Computer Science, Purdue University, West Lafayette, IN USA</nlm:aff></affiliation><affiliation><nlm:aff id="Aff85">Department of Biological Sciences, Purdue University, West Lafayette, IN USA</nlm:aff></affiliation></author><author><name sortKey="Ofer, Dan" sort="Ofer, Dan" uniqKey="Ofer D" first="Dan" last="Ofer">Dan Ofer</name><affiliation><nlm:aff id="Aff32">Department of Biological Chemistry, Institute of Life Sciences, The Hebrew University of Jerusalem, Jerusalem, Israel</nlm:aff></affiliation></author><author><name sortKey="Rappoport, Nadav" sort="Rappoport, Nadav" uniqKey="Rappoport N" first="Nadav" last="Rappoport">Nadav Rappoport</name><affiliation><nlm:aff id="Aff32">Department of Biological Chemistry, Institute of Life Sciences, The Hebrew University of Jerusalem, Jerusalem, Israel</nlm:aff></affiliation><affiliation><nlm:aff id="Aff33">School of Computer Science and Engineering, The Hebrew University of Jerusalem, Jerusalem, Israel</nlm:aff></affiliation></author><author><name sortKey="Stern, Amos" sort="Stern, Amos" uniqKey="Stern A" first="Amos" last="Stern">Amos Stern</name><affiliation><nlm:aff id="Aff32">Department of Biological Chemistry, Institute of Life Sciences, The Hebrew University of Jerusalem, Jerusalem, Israel</nlm:aff></affiliation><affiliation><nlm:aff id="Aff33">School of Computer Science and Engineering, The Hebrew University of Jerusalem, Jerusalem, Israel</nlm:aff></affiliation></author><author><name sortKey="Cibrian Uhalte, Elena" sort="Cibrian Uhalte, Elena" uniqKey="Cibrian Uhalte E" first="Elena" last="Cibrian-Uhalte">Elena Cibrian-Uhalte</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Denny, Paul" sort="Denny, Paul" uniqKey="Denny P" first="Paul" last="Denny">Paul Denny</name><affiliation><nlm:aff id="Aff35">Centre for Cardiovascular Genetics, Institute of Cardiovascular Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Foulger, Rebecca E" sort="Foulger, Rebecca E" uniqKey="Foulger R" first="Rebecca E." last="Foulger">Rebecca E. Foulger</name><affiliation><nlm:aff id="Aff35">Centre for Cardiovascular Genetics, Institute of Cardiovascular Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Hieta, Reija" sort="Hieta, Reija" uniqKey="Hieta R" first="Reija" last="Hieta">Reija Hieta</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Legge, Duncan" sort="Legge, Duncan" uniqKey="Legge D" first="Duncan" last="Legge">Duncan Legge</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Lovering, Ruth C" sort="Lovering, Ruth C" uniqKey="Lovering R" first="Ruth C." last="Lovering">Ruth C. Lovering</name><affiliation><nlm:aff id="Aff35">Centre for Cardiovascular Genetics, Institute of Cardiovascular Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Magrane, Michele" sort="Magrane, Michele" uniqKey="Magrane M" first="Michele" last="Magrane">Michele Magrane</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Melidoni, Anna N" sort="Melidoni, Anna N" uniqKey="Melidoni A" first="Anna N." last="Melidoni">Anna N. Melidoni</name><affiliation><nlm:aff id="Aff35">Centre for Cardiovascular Genetics, Institute of Cardiovascular Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Mutowo Meullenet, Prudence" sort="Mutowo Meullenet, Prudence" uniqKey="Mutowo Meullenet P" first="Prudence" last="Mutowo-Meullenet">Prudence Mutowo-Meullenet</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Pichler, Klemens" sort="Pichler, Klemens" uniqKey="Pichler K" first="Klemens" last="Pichler">Klemens Pichler</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Shypitsyna, Aleksandra" sort="Shypitsyna, Aleksandra" uniqKey="Shypitsyna A" first="Aleksandra" last="Shypitsyna">Aleksandra Shypitsyna</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Li, Biao" sort="Li, Biao" uniqKey="Li B" first="Biao" last="Li">Biao Li</name><affiliation><nlm:aff id="Aff2">Buck Institute for Research on Aging, Novato, CA USA</nlm:aff></affiliation></author><author><name sortKey="Zakeri, Pooya" sort="Zakeri, Pooya" uniqKey="Zakeri P" first="Pooya" last="Zakeri">Pooya Zakeri</name><affiliation><nlm:aff id="Aff36">Department of Electrical Engineering, STADIUS Center for Dynamical Systems, Signal Processing and Data Analytics, KU Leuven, Leuven, Belgium</nlm:aff></affiliation><affiliation><nlm:aff id="Aff37">iMinds Department Medical Information Technologies, Leuven, Belgium</nlm:aff></affiliation></author><author><name sortKey="Elshal, Sarah" sort="Elshal, Sarah" uniqKey="Elshal S" first="Sarah" last="Elshal">Sarah Elshal</name><affiliation><nlm:aff id="Aff36">Department of Electrical Engineering, STADIUS Center for Dynamical Systems, Signal Processing and Data Analytics, KU Leuven, Leuven, Belgium</nlm:aff></affiliation><affiliation><nlm:aff id="Aff37">iMinds Department Medical Information Technologies, Leuven, Belgium</nlm:aff></affiliation></author><author><name sortKey="Tranchevent, Leon Charles" sort="Tranchevent, Leon Charles" uniqKey="Tranchevent L" first="Léon-Charles" last="Tranchevent">Léon-Charles Tranchevent</name><affiliation><nlm:aff id="Aff38">Inserm UMR-S1052, CNRS UMR5286, Cancer Research Centre of Lyon, Lyon, France</nlm:aff></affiliation><affiliation><nlm:aff id="Aff39">Université de Lyon 1, Villeurbanne, France</nlm:aff></affiliation><affiliation><nlm:aff id="Aff40">Centre Léon Bérard, Lyon, France</nlm:aff></affiliation></author><author><name sortKey="Das, Sayoni" sort="Das, Sayoni" uniqKey="Das S" first="Sayoni" last="Das">Sayoni Das</name><affiliation><nlm:aff id="Aff41">Institute of Structural and Molecular Biology, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Dawson, Natalie L" sort="Dawson, Natalie L" uniqKey="Dawson N" first="Natalie L." last="Dawson">Natalie L. Dawson</name><affiliation><nlm:aff id="Aff41">Institute of Structural and Molecular Biology, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Lee, David" sort="Lee, David" uniqKey="Lee D" first="David" last="Lee">David Lee</name><affiliation><nlm:aff id="Aff41">Institute of Structural and Molecular Biology, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Lees, Jonathan G" sort="Lees, Jonathan G" uniqKey="Lees J" first="Jonathan G." last="Lees">Jonathan G. Lees</name><affiliation><nlm:aff id="Aff41">Institute of Structural and Molecular Biology, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Sillitoe, Ian" sort="Sillitoe, Ian" uniqKey="Sillitoe I" first="Ian" last="Sillitoe">Ian Sillitoe</name><affiliation><nlm:aff id="Aff41">Institute of Structural and Molecular Biology, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Bhat, Prajwal" sort="Bhat, Prajwal" uniqKey="Bhat P" first="Prajwal" last="Bhat">Prajwal Bhat</name><affiliation><nlm:aff id="Aff42">Cerenode Inc., Boston, MA USA</nlm:aff></affiliation></author><author><name sortKey="Nepusz, Tamas" sort="Nepusz, Tamas" uniqKey="Nepusz T" first="Tamás" last="Nepusz">Tamás Nepusz</name><affiliation><nlm:aff id="Aff43">Molde University College, Molde, Norway</nlm:aff></affiliation></author><author><name sortKey="Romero, Alfonso E" sort="Romero, Alfonso E" uniqKey="Romero A" first="Alfonso E." last="Romero">Alfonso E. Romero</name><affiliation><nlm:aff id="Aff44">Department of Computer Science, Centre for Systems and Synthetic Biology, Royal Holloway University of London, Egham, UK</nlm:aff></affiliation></author><author><name sortKey="Sasidharan, Rajkumar" sort="Sasidharan, Rajkumar" uniqKey="Sasidharan R" first="Rajkumar" last="Sasidharan">Rajkumar Sasidharan</name><affiliation><nlm:aff id="Aff45">Department of Molecular, Cell and Developmental Biology, University of California at Los Angeles, Los Angeles, CA USA</nlm:aff></affiliation></author><author><name sortKey="Yang, Haixuan" sort="Yang, Haixuan" uniqKey="Yang H" first="Haixuan" last="Yang">Haixuan Yang</name><affiliation><nlm:aff id="Aff46">School of Mathematics, Statistics and Applied Mathematics, National University of Ireland, Galway, Ireland</nlm:aff></affiliation></author><author><name sortKey="Paccanaro, Alberto" sort="Paccanaro, Alberto" uniqKey="Paccanaro A" first="Alberto" last="Paccanaro">Alberto Paccanaro</name><affiliation><nlm:aff id="Aff44">Department of Computer Science, Centre for Systems and Synthetic Biology, Royal Holloway University of London, Egham, UK</nlm:aff></affiliation></author><author><name sortKey="Gillis, Jesse" sort="Gillis, Jesse" uniqKey="Gillis J" first="Jesse" last="Gillis">Jesse Gillis</name><affiliation><nlm:aff id="Aff47">Stanley Institute for Cognitive Genomics Cold Spring Harbor Laboratory, New York, NY USA</nlm:aff></affiliation></author><author><name sortKey="Sede O Cortes, Adriana E" sort="Sede O Cortes, Adriana E" uniqKey="Sede O Cortes A" first="Adriana E." last="Sede O-Cortés">Adriana E. Sede O-Cortés</name><affiliation><nlm:aff id="Aff48">Graduate Program in Bioinformatics, University of British Columbia, Vancouver, Canada</nlm:aff></affiliation></author><author><name sortKey="Pavlidis, Paul" sort="Pavlidis, Paul" uniqKey="Pavlidis P" first="Paul" last="Pavlidis">Paul Pavlidis</name><affiliation><nlm:aff id="Aff49">Department of Psychiatry and Michael Smith Laboratories, University of British Columbia, Vancouver, Canada</nlm:aff></affiliation></author><author><name sortKey="Feng, Shou" sort="Feng, Shou" uniqKey="Feng S" first="Shou" last="Feng">Shou Feng</name><affiliation><nlm:aff id="Aff1">Department of Computer Science and Informatics, Indiana University, Bloomington, IN USA</nlm:aff></affiliation></author><author><name sortKey="Cejuela, Juan M" sort="Cejuela, Juan M" uniqKey="Cejuela J" first="Juan M." last="Cejuela">Juan M. Cejuela</name><affiliation><nlm:aff id="Aff50">Department for Bioinformatics and Computational Biology-I12, Technische Universität München, Garching, Germany</nlm:aff></affiliation></author><author><name sortKey="Goldberg, Tatyana" sort="Goldberg, Tatyana" uniqKey="Goldberg T" first="Tatyana" last="Goldberg">Tatyana Goldberg</name><affiliation><nlm:aff id="Aff50">Department for Bioinformatics and Computational Biology-I12, Technische Universität München, Garching, Germany</nlm:aff></affiliation></author><author><name sortKey="Hamp, Tobias" sort="Hamp, Tobias" uniqKey="Hamp T" first="Tobias" last="Hamp">Tobias Hamp</name><affiliation><nlm:aff id="Aff50">Department for Bioinformatics and Computational Biology-I12, Technische Universität München, Garching, Germany</nlm:aff></affiliation></author><author><name sortKey="Richter, Lothar" sort="Richter, Lothar" uniqKey="Richter L" first="Lothar" last="Richter">Lothar Richter</name><affiliation><nlm:aff id="Aff50">Department for Bioinformatics and Computational Biology-I12, Technische Universität München, Garching, Germany</nlm:aff></affiliation></author><author><name sortKey="Salamov, Asaf" sort="Salamov, Asaf" uniqKey="Salamov A" first="Asaf" last="Salamov">Asaf Salamov</name><affiliation><nlm:aff id="Aff51">DOE Joint Genome Institute, Walnut Creek, CA USA</nlm:aff></affiliation></author><author><name sortKey="Gabaldon, Toni" sort="Gabaldon, Toni" uniqKey="Gabaldon T" first="Toni" last="Gabaldon">Toni Gabaldon</name><affiliation><nlm:aff id="Aff52">Bioinformatics and Genomics, Centre for Genomic Regulation, Barcelona, Spain</nlm:aff></affiliation><affiliation><nlm:aff id="Aff53">Universitat Pompeu Fabra, Barcelona, Spain</nlm:aff></affiliation><affiliation><nlm:aff id="Aff54">Institució Catalana de Recerca i Estudis Avançats, Barcelona, Spain</nlm:aff></affiliation></author><author><name sortKey="Marcet Houben, Marina" sort="Marcet Houben, Marina" uniqKey="Marcet Houben M" first="Marina" last="Marcet-Houben">Marina Marcet-Houben</name><affiliation><nlm:aff id="Aff52">Bioinformatics and Genomics, Centre for Genomic Regulation, Barcelona, Spain</nlm:aff></affiliation><affiliation><nlm:aff id="Aff53">Universitat Pompeu Fabra, Barcelona, Spain</nlm:aff></affiliation></author><author><name sortKey="Supek, Fran" sort="Supek, Fran" uniqKey="Supek F" first="Fran" last="Supek">Fran Supek</name><affiliation><nlm:aff id="Aff53">Universitat Pompeu Fabra, Barcelona, Spain</nlm:aff></affiliation><affiliation><nlm:aff id="Aff55">Division of Electronics, Rudjer Boskovic Institute, Zagreb, Croatia</nlm:aff></affiliation><affiliation><nlm:aff id="Aff56">EMBL/CRG Systems Biology Research Unit, Centre for Genomic Regulation, Barcelona, Spain</nlm:aff></affiliation></author><author><name sortKey="Gong, Qingtian" sort="Gong, Qingtian" uniqKey="Gong Q" first="Qingtian" last="Gong">Qingtian Gong</name><affiliation><nlm:aff id="Aff57">State Key Laboratory of Genetic Engineering, Collaborative Innovation Center of Genetics and Development, Department of Biostatistics and Computational Biology, School of Life Science, Fudan University, Shanghai, China</nlm:aff></affiliation><affiliation><nlm:aff id="Aff58">Children’s Hospital of Fudan University, Shanghai, China</nlm:aff></affiliation></author><author><name sortKey="Ning, Wei" sort="Ning, Wei" uniqKey="Ning W" first="Wei" last="Ning">Wei Ning</name><affiliation><nlm:aff id="Aff57">State Key Laboratory of Genetic Engineering, Collaborative Innovation Center of Genetics and Development, Department of Biostatistics and Computational Biology, School of Life Science, Fudan University, Shanghai, China</nlm:aff></affiliation><affiliation><nlm:aff id="Aff58">Children’s Hospital of Fudan University, Shanghai, China</nlm:aff></affiliation></author><author><name sortKey="Zhou, Yuanpeng" sort="Zhou, Yuanpeng" uniqKey="Zhou Y" first="Yuanpeng" last="Zhou">Yuanpeng Zhou</name><affiliation><nlm:aff id="Aff57">State Key Laboratory of Genetic Engineering, Collaborative Innovation Center of Genetics and Development, Department of Biostatistics and Computational Biology, School of Life Science, Fudan University, Shanghai, China</nlm:aff></affiliation><affiliation><nlm:aff id="Aff58">Children’s Hospital of Fudan University, Shanghai, China</nlm:aff></affiliation></author><author><name sortKey="Tian, Weidong" sort="Tian, Weidong" uniqKey="Tian W" first="Weidong" last="Tian">Weidong Tian</name><affiliation><nlm:aff id="Aff57">State Key Laboratory of Genetic Engineering, Collaborative Innovation Center of Genetics and Development, Department of Biostatistics and Computational Biology, School of Life Science, Fudan University, Shanghai, China</nlm:aff></affiliation><affiliation><nlm:aff id="Aff58">Children’s Hospital of Fudan University, Shanghai, China</nlm:aff></affiliation></author><author><name sortKey="Falda, Marco" sort="Falda, Marco" uniqKey="Falda M" first="Marco" last="Falda">Marco Falda</name><affiliation><nlm:aff id="Aff59">Department of Molecular Medicine, University of Padua, Padua, Italy</nlm:aff></affiliation></author><author><name sortKey="Fontana, Paolo" sort="Fontana, Paolo" uniqKey="Fontana P" first="Paolo" last="Fontana">Paolo Fontana</name><affiliation><nlm:aff id="Aff60">Research and Innovation Center, Edmund Mach Foundation, San Michele all’Adige, Italy</nlm:aff></affiliation></author><author><name sortKey="Lavezzo, Enrico" sort="Lavezzo, Enrico" uniqKey="Lavezzo E" first="Enrico" last="Lavezzo">Enrico Lavezzo</name><affiliation><nlm:aff id="Aff59">Department of Molecular Medicine, University of Padua, Padua, Italy</nlm:aff></affiliation></author><author><name sortKey="Toppo, Stefano" sort="Toppo, Stefano" uniqKey="Toppo S" first="Stefano" last="Toppo">Stefano Toppo</name><affiliation><nlm:aff id="Aff59">Department of Molecular Medicine, University of Padua, Padua, Italy</nlm:aff></affiliation></author><author><name sortKey="Ferrari, Carlo" sort="Ferrari, Carlo" uniqKey="Ferrari C" first="Carlo" last="Ferrari">Carlo Ferrari</name><affiliation><nlm:aff id="Aff61">Department of Information Engineering, University of Padua, Padova, Italy</nlm:aff></affiliation></author><author><name sortKey="Giollo, Manuel" sort="Giollo, Manuel" uniqKey="Giollo M" first="Manuel" last="Giollo">Manuel Giollo</name><affiliation><nlm:aff id="Aff61">Department of Information Engineering, University of Padua, Padova, Italy</nlm:aff></affiliation><affiliation><nlm:aff id="Aff84">Department of Biomedical Sciences, University of Padua, Padova, Italy</nlm:aff></affiliation></author><author><name sortKey="Piovesan, Damiano" sort="Piovesan, Damiano" uniqKey="Piovesan D" first="Damiano" last="Piovesan">Damiano Piovesan</name><affiliation><nlm:aff id="Aff61">Department of Information Engineering, University of Padua, Padova, Italy</nlm:aff></affiliation></author><author><name sortKey="Tosatto, Silvio C E" sort="Tosatto, Silvio C E" uniqKey="Tosatto S" first="Silvio C. E." last="Tosatto">Silvio C. E. Tosatto</name><affiliation><nlm:aff id="Aff61">Department of Information Engineering, University of Padua, Padova, Italy</nlm:aff></affiliation></author><author><name sortKey="Del Pozo, Angela" sort="Del Pozo, Angela" uniqKey="Del Pozo A" first="Angela" last="Del Pozo">Angela Del Pozo</name><affiliation><nlm:aff id="Aff62">Instituto De Genetica Medica y Molecular, Hospital Universitario de La Paz, Madrid, Spain</nlm:aff></affiliation></author><author><name sortKey="Fernandez, Jose M" sort="Fernandez, Jose M" uniqKey="Fernandez J" first="José M." last="Fernández">José M. Fernández</name><affiliation><nlm:aff id="Aff63">Spanish National Bioinformatics Institute, Spanish National Cancer Research Institute, Madrid, Spain</nlm:aff></affiliation></author><author><name sortKey="Maietta, Paolo" sort="Maietta, Paolo" uniqKey="Maietta P" first="Paolo" last="Maietta">Paolo Maietta</name><affiliation><nlm:aff id="Aff64">Structural and Computational Biology Programme, Spanish National Cancer Research Institute, Madrid, Spain</nlm:aff></affiliation></author><author><name sortKey="Valencia, Alfonso" sort="Valencia, Alfonso" uniqKey="Valencia A" first="Alfonso" last="Valencia">Alfonso Valencia</name><affiliation><nlm:aff id="Aff64">Structural and Computational Biology Programme, Spanish National Cancer Research Institute, Madrid, Spain</nlm:aff></affiliation></author><author><name sortKey="Tress, Michael L" sort="Tress, Michael L" uniqKey="Tress M" first="Michael L." last="Tress">Michael L. Tress</name><affiliation><nlm:aff id="Aff64">Structural and Computational Biology Programme, Spanish National Cancer Research Institute, Madrid, Spain</nlm:aff></affiliation></author><author><name sortKey="Benso, Alfredo" sort="Benso, Alfredo" uniqKey="Benso A" first="Alfredo" last="Benso">Alfredo Benso</name><affiliation><nlm:aff id="Aff65">Control and Computer Engineering Department, Politecnico di Torino, Torino, Italy</nlm:aff></affiliation></author><author><name sortKey="Di Carlo, Stefano" sort="Di Carlo, Stefano" uniqKey="Di Carlo S" first="Stefano" last="Di Carlo">Stefano Di Carlo</name><affiliation><nlm:aff id="Aff65">Control and Computer Engineering Department, Politecnico di Torino, Torino, Italy</nlm:aff></affiliation></author><author><name sortKey="Politano, Gianfranco" sort="Politano, Gianfranco" uniqKey="Politano G" first="Gianfranco" last="Politano">Gianfranco Politano</name><affiliation><nlm:aff id="Aff65">Control and Computer Engineering Department, Politecnico di Torino, Torino, Italy</nlm:aff></affiliation></author><author><name sortKey="Savino, Alessandro" sort="Savino, Alessandro" uniqKey="Savino A" first="Alessandro" last="Savino">Alessandro Savino</name><affiliation><nlm:aff id="Aff65">Control and Computer Engineering Department, Politecnico di Torino, Torino, Italy</nlm:aff></affiliation></author><author><name sortKey="Rehman, Hafeez Ur" sort="Rehman, Hafeez Ur" uniqKey="Rehman H" first="Hafeez Ur" last="Rehman">Hafeez Ur Rehman</name><affiliation><nlm:aff id="Aff66">National University of Computer & Emerging Sciences, Islamabad, Pakistan</nlm:aff></affiliation></author><author><name sortKey="Re, Matteo" sort="Re, Matteo" uniqKey="Re M" first="Matteo" last="Re">Matteo Re</name><affiliation><nlm:aff id="Aff67">Anacleto Lab, Dipartimento di informatica, Università degli Studi di Milano, Milan, Italy</nlm:aff></affiliation></author><author><name sortKey="Mesiti, Marco" sort="Mesiti, Marco" uniqKey="Mesiti M" first="Marco" last="Mesiti">Marco Mesiti</name><affiliation><nlm:aff id="Aff67">Anacleto Lab, Dipartimento di informatica, Università degli Studi di Milano, Milan, Italy</nlm:aff></affiliation></author><author><name sortKey="Valentini, Giorgio" sort="Valentini, Giorgio" uniqKey="Valentini G" first="Giorgio" last="Valentini">Giorgio Valentini</name><affiliation><nlm:aff id="Aff67">Anacleto Lab, Dipartimento di informatica, Università degli Studi di Milano, Milan, Italy</nlm:aff></affiliation></author><author><name sortKey="Bargsten, Joachim W" sort="Bargsten, Joachim W" uniqKey="Bargsten J" first="Joachim W." last="Bargsten">Joachim W. Bargsten</name><affiliation><nlm:aff id="Aff68">Applied Bioinformatics, Bioscience, Wageningen University and Research Centre, Wageningen, Netherlands</nlm:aff></affiliation></author><author><name sortKey="Van Dijk, Aalt D J" sort="Van Dijk, Aalt D J" uniqKey="Van Dijk A" first="Aalt D. J." last="Van Dijk">Aalt D. J. Van Dijk</name><affiliation><nlm:aff id="Aff68">Applied Bioinformatics, Bioscience, Wageningen University and Research Centre, Wageningen, Netherlands</nlm:aff></affiliation><affiliation><nlm:aff id="Aff69">Biometris, Wageningen University, Wageningen, Netherlands</nlm:aff></affiliation></author><author><name sortKey="Gemovic, Branislava" sort="Gemovic, Branislava" uniqKey="Gemovic B" first="Branislava" last="Gemovic">Branislava Gemovic</name><affiliation><nlm:aff id="Aff70">Center for Multidisciplinary Research, Institute of Nuclear Sciences Vinca, University of Belgrade, Belgrade, Serbia</nlm:aff></affiliation></author><author><name sortKey="Glisic, Sanja" sort="Glisic, Sanja" uniqKey="Glisic S" first="Sanja" last="Glisic">Sanja Glisic</name><affiliation><nlm:aff id="Aff70">Center for Multidisciplinary Research, Institute of Nuclear Sciences Vinca, University of Belgrade, Belgrade, Serbia</nlm:aff></affiliation></author><author><name sortKey="Perovic, Vladmir" sort="Perovic, Vladmir" uniqKey="Perovic V" first="Vladmir" last="Perovic">Vladmir Perovic</name><affiliation><nlm:aff id="Aff70">Center for Multidisciplinary Research, Institute of Nuclear Sciences Vinca, University of Belgrade, Belgrade, Serbia</nlm:aff></affiliation></author><author><name sortKey="Veljkovic, Veljko" sort="Veljkovic, Veljko" uniqKey="Veljkovic V" first="Veljko" last="Veljkovic">Veljko Veljkovic</name><affiliation><nlm:aff id="Aff70">Center for Multidisciplinary Research, Institute of Nuclear Sciences Vinca, University of Belgrade, Belgrade, Serbia</nlm:aff></affiliation></author><author><name sortKey="Veljkovic, Nevena" sort="Veljkovic, Nevena" uniqKey="Veljkovic N" first="Nevena" last="Veljkovic">Nevena Veljkovic</name><affiliation><nlm:aff id="Aff70">Center for Multidisciplinary Research, Institute of Nuclear Sciences Vinca, University of Belgrade, Belgrade, Serbia</nlm:aff></affiliation></author><author><name sortKey="Almeida E Silva, Danillo C" sort="Almeida E Silva, Danillo C" uniqKey="Almeida E Silva D" first="Danillo C." last="Almeida-E-Silva">Danillo C. Almeida-E-Silva</name><affiliation><nlm:aff id="Aff71">Department of Computing and Mathematics FFCLRP-USP, University of Sao Paulo, Ribeirao Preto, Brazil</nlm:aff></affiliation></author><author><name sortKey="Vencio, Ricardo Z N" sort="Vencio, Ricardo Z N" uniqKey="Vencio R" first="Ricardo Z. N." last="Vencio">Ricardo Z. N. Vencio</name><affiliation><nlm:aff id="Aff71">Department of Computing and Mathematics FFCLRP-USP, University of Sao Paulo, Ribeirao Preto, Brazil</nlm:aff></affiliation></author><author><name sortKey="Sharan, Malvika" sort="Sharan, Malvika" uniqKey="Sharan M" first="Malvika" last="Sharan">Malvika Sharan</name><affiliation><nlm:aff id="Aff72">Institute for Molecular Infection Biology, University of Würzburg, Würzburg, Germany</nlm:aff></affiliation></author><author><name sortKey="Vogel, Jorg" sort="Vogel, Jorg" uniqKey="Vogel J" first="Jörg" last="Vogel">Jörg Vogel</name><affiliation><nlm:aff id="Aff72">Institute for Molecular Infection Biology, University of Würzburg, Würzburg, Germany</nlm:aff></affiliation></author><author><name sortKey="Kansakar, Lakesh" sort="Kansakar, Lakesh" uniqKey="Kansakar L" first="Lakesh" last="Kansakar">Lakesh Kansakar</name><affiliation><nlm:aff id="Aff73">Department of Computer and Information Sciences, Temple University, Philadelphia, PA USA</nlm:aff></affiliation></author><author><name sortKey="Zhang, Shanshan" sort="Zhang, Shanshan" uniqKey="Zhang S" first="Shanshan" last="Zhang">Shanshan Zhang</name><affiliation><nlm:aff id="Aff73">Department of Computer and Information Sciences, Temple University, Philadelphia, PA USA</nlm:aff></affiliation></author><author><name sortKey="Vucetic, Slobodan" sort="Vucetic, Slobodan" uniqKey="Vucetic S" first="Slobodan" last="Vucetic">Slobodan Vucetic</name><affiliation><nlm:aff id="Aff73">Department of Computer and Information Sciences, Temple University, Philadelphia, PA USA</nlm:aff></affiliation></author><author><name sortKey="Wang, Zheng" sort="Wang, Zheng" uniqKey="Wang Z" first="Zheng" last="Wang">Zheng Wang</name><affiliation><nlm:aff id="Aff74">University of Southern Mississippi, Hattiesburg, MS USA</nlm:aff></affiliation></author><author><name sortKey="Sternberg, Michael J E" sort="Sternberg, Michael J E" uniqKey="Sternberg M" first="Michael J. E." last="Sternberg">Michael J. E. Sternberg</name><affiliation><nlm:aff id="Aff34">Centre for Integrative Systems Biology and Bioinformatics, Department of Life Sciences, Imperial College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Wass, Mark N" sort="Wass, Mark N" uniqKey="Wass M" first="Mark N." last="Wass">Mark N. Wass</name><affiliation><nlm:aff id="Aff75">School of Biosciences, University of Kent, Canterbury, Kent, UK</nlm:aff></affiliation></author><author><name sortKey="Huntley, Rachael P" sort="Huntley, Rachael P" uniqKey="Huntley R" first="Rachael P." last="Huntley">Rachael P. Huntley</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Martin, Maria J" sort="Martin, Maria J" uniqKey="Martin M" first="Maria J." last="Martin">Maria J. Martin</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="O Onovan, Claire" sort="O Onovan, Claire" uniqKey="O Onovan C" first="Claire" last="O Onovan">Claire O Onovan</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Robinson, Peter N" sort="Robinson, Peter N" uniqKey="Robinson P" first="Peter N." last="Robinson">Peter N. Robinson</name><affiliation><nlm:aff id="Aff76">Institut für Medizinische Genetik und Humangenetik, Charité - Universitätsmedizin Berlin, Berlin, Germany</nlm:aff></affiliation></author><author><name sortKey="Moreau, Yves" sort="Moreau, Yves" uniqKey="Moreau Y" first="Yves" last="Moreau">Yves Moreau</name><affiliation><nlm:aff id="Aff77">Department of Electrical Engineering ESAT-SCD and IBBT-KU Leuven Future Health Department, Katholieke Universiteit Leuven, Leuven, Belgium</nlm:aff></affiliation></author><author><name sortKey="Tramontano, Anna" sort="Tramontano, Anna" uniqKey="Tramontano A" first="Anna" last="Tramontano">Anna Tramontano</name><affiliation><nlm:aff id="Aff5">University of Rome, La Sapienza, Rome, Italy</nlm:aff></affiliation></author><author><name sortKey="Babbitt, Patricia C" sort="Babbitt, Patricia C" uniqKey="Babbitt P" first="Patricia C." last="Babbitt">Patricia C. Babbitt</name><affiliation><nlm:aff id="Aff78">California Institute for Quantitative Biosciences, University of California San Francisco, San Francisco, CA USA</nlm:aff></affiliation></author><author><name sortKey="Brenner, Steven E" sort="Brenner, Steven E" uniqKey="Brenner S" first="Steven E." last="Brenner">Steven E. Brenner</name><affiliation><nlm:aff id="Aff17">Department of Plant and Microbial Biology, University of California Berkeley, Berkeley, CA USA</nlm:aff></affiliation></author><author><name sortKey="Linial, Michal" sort="Linial, Michal" uniqKey="Linial M" first="Michal" last="Linial">Michal Linial</name><affiliation><nlm:aff id="Aff79">Department of Chemical Biology, The Hebrew University of Jerusalem, Jerusalem, Israel</nlm:aff></affiliation></author><author><name sortKey="Orengo, Christine A" sort="Orengo, Christine A" uniqKey="Orengo C" first="Christine A." last="Orengo">Christine A. Orengo</name><affiliation><nlm:aff id="Aff41">Institute of Structural and Molecular Biology, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Rost, Burkhard" sort="Rost, Burkhard" uniqKey="Rost B" first="Burkhard" last="Rost">Burkhard Rost</name><affiliation><nlm:aff id="Aff50">Department for Bioinformatics and Computational Biology-I12, Technische Universität München, Garching, Germany</nlm:aff></affiliation></author><author><name sortKey="Greene, Casey S" sort="Greene, Casey S" uniqKey="Greene C" first="Casey S." last="Greene">Casey S. Greene</name><affiliation><nlm:aff id="Aff80">Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania, Philadelphia, PA USA</nlm:aff></affiliation></author><author><name sortKey="Mooney, Sean D" sort="Mooney, Sean D" uniqKey="Mooney S" first="Sean D." last="Mooney">Sean D. Mooney</name><affiliation><nlm:aff id="Aff81">Department of Biomedical Informatics and Medical Education, University of Washington, Seattle, WA USA</nlm:aff></affiliation></author><author><name sortKey="Friedberg, Iddo" sort="Friedberg, Iddo" uniqKey="Friedberg I" first="Iddo" last="Friedberg">Iddo Friedberg</name><affiliation><nlm:aff id="Aff4">Department of Microbiology, Miami University, Oxford, OH USA</nlm:aff></affiliation><affiliation><nlm:aff id="Aff82">Department of Computer Science, Miami University, Oxford, OH USA</nlm:aff></affiliation></author><author><name sortKey="Radivojac, Predrag" sort="Radivojac, Predrag" uniqKey="Radivojac P" first="Predrag" last="Radivojac">Predrag Radivojac</name><affiliation><nlm:aff id="Aff1">Department of Computer Science and Informatics, Indiana University, Bloomington, IN USA</nlm:aff></affiliation></author></titleStmt><publicationStmt><idno type="wicri:source">PMC</idno><idno type="pmid">27604469</idno><idno type="pmc">5015320</idno><idno type="url">http://www.ncbi.nlm.nih.gov/pmc/articles/PMC5015320</idno><idno type="RBID">PMC:5015320</idno><idno type="doi">10.1186/s13059-016-1037-6</idno><date when="2016">2016</date><idno type="wicri:Area/Pmc/Corpus">000D08</idno><idno type="wicri:explorRef" wicri:stream="Pmc" wicri:step="Corpus" wicri:corpus="PMC">000D08</idno></publicationStmt><sourceDesc><biblStruct><analytic><title xml:lang="en" level="a" type="main">An expanded evaluation of protein function prediction methods shows an improvement in accuracy</title><author><name sortKey="Jiang, Yuxiang" sort="Jiang, Yuxiang" uniqKey="Jiang Y" first="Yuxiang" last="Jiang">Yuxiang Jiang</name><affiliation><nlm:aff id="Aff1">Department of Computer Science and Informatics, Indiana University, Bloomington, IN USA</nlm:aff></affiliation></author><author><name sortKey="Oron, Tal Ronnen" sort="Oron, Tal Ronnen" uniqKey="Oron T" first="Tal Ronnen" last="Oron">Tal Ronnen Oron</name><affiliation><nlm:aff id="Aff2">Buck Institute for Research on Aging, Novato, CA USA</nlm:aff></affiliation></author><author><name sortKey="Clark, Wyatt T" sort="Clark, Wyatt T" uniqKey="Clark W" first="Wyatt T." last="Clark">Wyatt T. Clark</name><affiliation><nlm:aff id="Aff3">Department of Molecular Biophysics and Biochemistry, Yale University, New Haven, CT USA</nlm:aff></affiliation></author><author><name sortKey="Bankapur, Asma R" sort="Bankapur, Asma R" uniqKey="Bankapur A" first="Asma R." last="Bankapur">Asma R. Bankapur</name><affiliation><nlm:aff id="Aff4">Department of Microbiology, Miami University, Oxford, OH USA</nlm:aff></affiliation></author><author><name sortKey="D Ndrea, Daniel" sort="D Ndrea, Daniel" uniqKey="D Ndrea D" first="Daniel" last="D Ndrea">Daniel D Ndrea</name><affiliation><nlm:aff id="Aff5">University of Rome, La Sapienza, Rome, Italy</nlm:aff></affiliation></author><author><name sortKey="Lepore, Rosalba" sort="Lepore, Rosalba" uniqKey="Lepore R" first="Rosalba" last="Lepore">Rosalba Lepore</name><affiliation><nlm:aff id="Aff5">University of Rome, La Sapienza, Rome, Italy</nlm:aff></affiliation></author><author><name sortKey="Funk, Christopher S" sort="Funk, Christopher S" uniqKey="Funk C" first="Christopher S." last="Funk">Christopher S. Funk</name><affiliation><nlm:aff id="Aff6">Computational Bioscience Program, University of Colorado School of Medicine, Aurora, CO USA</nlm:aff></affiliation></author><author><name sortKey="Kahanda, Indika" sort="Kahanda, Indika" uniqKey="Kahanda I" first="Indika" last="Kahanda">Indika Kahanda</name><affiliation><nlm:aff id="Aff7">Department of Computer Science, Colorado State University, Fort Collins, CO USA</nlm:aff></affiliation></author><author><name sortKey="Verspoor, Karin M" sort="Verspoor, Karin M" uniqKey="Verspoor K" first="Karin M." last="Verspoor">Karin M. Verspoor</name><affiliation><nlm:aff id="Aff8">Department of Computing and Information Systems, University of Melbourne, Parkville, Victoria, Australia</nlm:aff></affiliation><affiliation><nlm:aff id="Aff9">Health and Biomedical Informatics Centre, University of Melbourne, Parkville, Victoria, Australia</nlm:aff></affiliation></author><author><name sortKey="Ben Hur, Asa" sort="Ben Hur, Asa" uniqKey="Ben Hur A" first="Asa" last="Ben-Hur">Asa Ben-Hur</name><affiliation><nlm:aff id="Aff7">Department of Computer Science, Colorado State University, Fort Collins, CO USA</nlm:aff></affiliation></author><author><name sortKey="Koo, Da Chen Emily" sort="Koo, Da Chen Emily" uniqKey="Koo D" first="Da Chen Emily" last="Koo">Da Chen Emily Koo</name><affiliation><nlm:aff id="Aff10">Department of Biology, New York University, New York, NY USA</nlm:aff></affiliation></author><author><name sortKey="Penfold Brown, Duncan" sort="Penfold Brown, Duncan" uniqKey="Penfold Brown D" first="Duncan" last="Penfold-Brown">Duncan Penfold-Brown</name><affiliation><nlm:aff id="Aff11">Social Media and Political Participation Lab, New York University, New York, NY USA</nlm:aff></affiliation><affiliation><nlm:aff id="Aff12">CY Data Science, New York, NY USA</nlm:aff></affiliation></author><author><name sortKey="Shasha, Dennis" sort="Shasha, Dennis" uniqKey="Shasha D" first="Dennis" last="Shasha">Dennis Shasha</name><affiliation><nlm:aff id="Aff13">Department of Computer Science, New York University, New York, NY USA</nlm:aff></affiliation></author><author><name sortKey="Youngs, Noah" sort="Youngs, Noah" uniqKey="Youngs N" first="Noah" last="Youngs">Noah Youngs</name><affiliation><nlm:aff id="Aff12">CY Data Science, New York, NY USA</nlm:aff></affiliation><affiliation><nlm:aff id="Aff13">Department of Computer Science, New York University, New York, NY USA</nlm:aff></affiliation><affiliation><nlm:aff id="Aff14">Simons Center for Data Analysis, New York, NY USA</nlm:aff></affiliation></author><author><name sortKey="Bonneau, Richard" sort="Bonneau, Richard" uniqKey="Bonneau R" first="Richard" last="Bonneau">Richard Bonneau</name><affiliation><nlm:aff id="Aff13">Department of Computer Science, New York University, New York, NY USA</nlm:aff></affiliation><affiliation><nlm:aff id="Aff14">Simons Center for Data Analysis, New York, NY USA</nlm:aff></affiliation><affiliation><nlm:aff id="Aff15">Center for Genomics and Systems Biology, Department of Biology, New York University, New York, NY USA</nlm:aff></affiliation></author><author><name sortKey="Lin, Alexandra" sort="Lin, Alexandra" uniqKey="Lin A" first="Alexandra" last="Lin">Alexandra Lin</name><affiliation><nlm:aff id="Aff16">Department of Electrical Engineering and Computer Sciences, University of California Berkeley, Berkeley, CA USA</nlm:aff></affiliation></author><author><name sortKey="Sahraeian, Sayed M E" sort="Sahraeian, Sayed M E" uniqKey="Sahraeian S" first="Sayed M. E." last="Sahraeian">Sayed M. E. Sahraeian</name><affiliation><nlm:aff id="Aff17">Department of Plant and Microbial Biology, University of California Berkeley, Berkeley, CA USA</nlm:aff></affiliation></author><author><name sortKey="Martelli, Pier Luigi" sort="Martelli, Pier Luigi" uniqKey="Martelli P" first="Pier Luigi" last="Martelli">Pier Luigi Martelli</name><affiliation><nlm:aff id="Aff18">Biocomputing Group, BiGeA, University of Bologna, Bologna, Italy</nlm:aff></affiliation></author><author><name sortKey="Profiti, Giuseppe" sort="Profiti, Giuseppe" uniqKey="Profiti G" first="Giuseppe" last="Profiti">Giuseppe Profiti</name><affiliation><nlm:aff id="Aff18">Biocomputing Group, BiGeA, University of Bologna, Bologna, Italy</nlm:aff></affiliation></author><author><name sortKey="Casadio, Rita" sort="Casadio, Rita" uniqKey="Casadio R" first="Rita" last="Casadio">Rita Casadio</name><affiliation><nlm:aff id="Aff18">Biocomputing Group, BiGeA, University of Bologna, Bologna, Italy</nlm:aff></affiliation></author><author><name sortKey="Cao, Renzhi" sort="Cao, Renzhi" uniqKey="Cao R" first="Renzhi" last="Cao">Renzhi Cao</name><affiliation><nlm:aff id="Aff19">Computer Science Department, University of Missouri, Columbia, MO USA</nlm:aff></affiliation></author><author><name sortKey="Zhong, Zhaolong" sort="Zhong, Zhaolong" uniqKey="Zhong Z" first="Zhaolong" last="Zhong">Zhaolong Zhong</name><affiliation><nlm:aff id="Aff19">Computer Science Department, University of Missouri, Columbia, MO USA</nlm:aff></affiliation></author><author><name sortKey="Cheng, Jianlin" sort="Cheng, Jianlin" uniqKey="Cheng J" first="Jianlin" last="Cheng">Jianlin Cheng</name><affiliation><nlm:aff id="Aff19">Computer Science Department, University of Missouri, Columbia, MO USA</nlm:aff></affiliation></author><author><name sortKey="Altenhoff, Adrian" sort="Altenhoff, Adrian" uniqKey="Altenhoff A" first="Adrian" last="Altenhoff">Adrian Altenhoff</name><affiliation><nlm:aff id="Aff20">ETH Zurich, Zurich, Switzerland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff21">Swiss Institute of Bioinformatics, Zurich, Switzerland</nlm:aff></affiliation></author><author><name sortKey="Skunca, Nives" sort="Skunca, Nives" uniqKey="Skunca N" first="Nives" last="Skunca">Nives Skunca</name><affiliation><nlm:aff id="Aff20">ETH Zurich, Zurich, Switzerland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff21">Swiss Institute of Bioinformatics, Zurich, Switzerland</nlm:aff></affiliation></author><author><name sortKey="Dessimoz, Christophe" sort="Dessimoz, Christophe" uniqKey="Dessimoz C" first="Christophe" last="Dessimoz">Christophe Dessimoz</name><affiliation><nlm:aff id="Aff22">Bioinformatics Group, Department of Computer Science, University College London, London, UK</nlm:aff></affiliation><affiliation><nlm:aff id="Aff87">University of Lausanne, Lausanne, Switzerland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff88">Swiss Institute of Bioinformatics, Lausanne, Switzerland</nlm:aff></affiliation></author><author><name sortKey="Dogan, Tunca" sort="Dogan, Tunca" uniqKey="Dogan T" first="Tunca" last="Dogan">Tunca Dogan</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Hakala, Kai" sort="Hakala, Kai" uniqKey="Hakala K" first="Kai" last="Hakala">Kai Hakala</name><affiliation><nlm:aff id="Aff24">Department of Information Technology, University of Turku, Turku, Finland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff25">University of Turku Graduate School, University of Turku, Turku, Finland</nlm:aff></affiliation></author><author><name sortKey="Kaewphan, Suwisa" sort="Kaewphan, Suwisa" uniqKey="Kaewphan S" first="Suwisa" last="Kaewphan">Suwisa Kaewphan</name><affiliation><nlm:aff id="Aff24">Department of Information Technology, University of Turku, Turku, Finland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff25">University of Turku Graduate School, University of Turku, Turku, Finland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff26">Turku Centre for Computer Science, Turku, Finland</nlm:aff></affiliation></author><author><name sortKey="Mehryary, Farrokh" sort="Mehryary, Farrokh" uniqKey="Mehryary F" first="Farrokh" last="Mehryary">Farrokh Mehryary</name><affiliation><nlm:aff id="Aff24">Department of Information Technology, University of Turku, Turku, Finland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff25">University of Turku Graduate School, University of Turku, Turku, Finland</nlm:aff></affiliation></author><author><name sortKey="Salakoski, Tapio" sort="Salakoski, Tapio" uniqKey="Salakoski T" first="Tapio" last="Salakoski">Tapio Salakoski</name><affiliation><nlm:aff id="Aff24">Department of Information Technology, University of Turku, Turku, Finland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff26">Turku Centre for Computer Science, Turku, Finland</nlm:aff></affiliation></author><author><name sortKey="Ginter, Filip" sort="Ginter, Filip" uniqKey="Ginter F" first="Filip" last="Ginter">Filip Ginter</name><affiliation><nlm:aff id="Aff24">Department of Information Technology, University of Turku, Turku, Finland</nlm:aff></affiliation></author><author><name sortKey="Fang, Hai" sort="Fang, Hai" uniqKey="Fang H" first="Hai" last="Fang">Hai Fang</name><affiliation><nlm:aff id="Aff27">University of Bristol, Bristol, UK</nlm:aff></affiliation></author><author><name sortKey="Smithers, Ben" sort="Smithers, Ben" uniqKey="Smithers B" first="Ben" last="Smithers">Ben Smithers</name><affiliation><nlm:aff id="Aff27">University of Bristol, Bristol, UK</nlm:aff></affiliation></author><author><name sortKey="Oates, Matt" sort="Oates, Matt" uniqKey="Oates M" first="Matt" last="Oates">Matt Oates</name><affiliation><nlm:aff id="Aff27">University of Bristol, Bristol, UK</nlm:aff></affiliation></author><author><name sortKey="Gough, Julian" sort="Gough, Julian" uniqKey="Gough J" first="Julian" last="Gough">Julian Gough</name><affiliation><nlm:aff id="Aff27">University of Bristol, Bristol, UK</nlm:aff></affiliation></author><author><name sortKey="Toronen, Petri" sort="Toronen, Petri" uniqKey="Toronen P" first="Petri" last="Törönen">Petri Törönen</name><affiliation><nlm:aff id="Aff28">Institute of Biotechnology, University of Helsinki, Helsinki, Finland</nlm:aff></affiliation></author><author><name sortKey="Koskinen, Patrik" sort="Koskinen, Patrik" uniqKey="Koskinen P" first="Patrik" last="Koskinen">Patrik Koskinen</name><affiliation><nlm:aff id="Aff28">Institute of Biotechnology, University of Helsinki, Helsinki, Finland</nlm:aff></affiliation></author><author><name sortKey="Holm, Liisa" sort="Holm, Liisa" uniqKey="Holm L" first="Liisa" last="Holm">Liisa Holm</name><affiliation><nlm:aff id="Aff28">Institute of Biotechnology, University of Helsinki, Helsinki, Finland</nlm:aff></affiliation><affiliation><nlm:aff id="Aff86">Department of Biological and Environmental Sciences, Universitity of Helsinki, Helsinki, Finland</nlm:aff></affiliation></author><author><name sortKey="Chen, Ching Tai" sort="Chen, Ching Tai" uniqKey="Chen C" first="Ching-Tai" last="Chen">Ching-Tai Chen</name><affiliation><nlm:aff id="Aff29">Institute of Information Science, Academia Sinica, Taipei, Taiwan</nlm:aff></affiliation></author><author><name sortKey="Hsu, Wen Lian" sort="Hsu, Wen Lian" uniqKey="Hsu W" first="Wen-Lian" last="Hsu">Wen-Lian Hsu</name><affiliation><nlm:aff id="Aff29">Institute of Information Science, Academia Sinica, Taipei, Taiwan</nlm:aff></affiliation></author><author><name sortKey="Bryson, Kevin" sort="Bryson, Kevin" uniqKey="Bryson K" first="Kevin" last="Bryson">Kevin Bryson</name><affiliation><nlm:aff id="Aff22">Bioinformatics Group, Department of Computer Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Cozzetto, Domenico" sort="Cozzetto, Domenico" uniqKey="Cozzetto D" first="Domenico" last="Cozzetto">Domenico Cozzetto</name><affiliation><nlm:aff id="Aff22">Bioinformatics Group, Department of Computer Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Minneci, Federico" sort="Minneci, Federico" uniqKey="Minneci F" first="Federico" last="Minneci">Federico Minneci</name><affiliation><nlm:aff id="Aff22">Bioinformatics Group, Department of Computer Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Jones, David T" sort="Jones, David T" uniqKey="Jones D" first="David T." last="Jones">David T. Jones</name><affiliation><nlm:aff id="Aff22">Bioinformatics Group, Department of Computer Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Chapman, Samuel" sort="Chapman, Samuel" uniqKey="Chapman S" first="Samuel" last="Chapman">Samuel Chapman</name><affiliation><nlm:aff id="Aff30">Department of Computational Science and Engineering, North Carolina A&T State University, Greensboro, NC USA</nlm:aff></affiliation></author><author><name sortKey="Bkc, Dukka" sort="Bkc, Dukka" uniqKey="Bkc D" first="Dukka" last="Bkc">Dukka Bkc</name><affiliation><nlm:aff id="Aff30">Department of Computational Science and Engineering, North Carolina A&T State University, Greensboro, NC USA</nlm:aff></affiliation></author><author><name sortKey="Khan, Ishita K" sort="Khan, Ishita K" uniqKey="Khan I" first="Ishita K." last="Khan">Ishita K. Khan</name><affiliation><nlm:aff id="Aff31">Department of Computer Science, Purdue University, West Lafayette, IN USA</nlm:aff></affiliation></author><author><name sortKey="Kihara, Daisuke" sort="Kihara, Daisuke" uniqKey="Kihara D" first="Daisuke" last="Kihara">Daisuke Kihara</name><affiliation><nlm:aff id="Aff31">Department of Computer Science, Purdue University, West Lafayette, IN USA</nlm:aff></affiliation><affiliation><nlm:aff id="Aff85">Department of Biological Sciences, Purdue University, West Lafayette, IN USA</nlm:aff></affiliation></author><author><name sortKey="Ofer, Dan" sort="Ofer, Dan" uniqKey="Ofer D" first="Dan" last="Ofer">Dan Ofer</name><affiliation><nlm:aff id="Aff32">Department of Biological Chemistry, Institute of Life Sciences, The Hebrew University of Jerusalem, Jerusalem, Israel</nlm:aff></affiliation></author><author><name sortKey="Rappoport, Nadav" sort="Rappoport, Nadav" uniqKey="Rappoport N" first="Nadav" last="Rappoport">Nadav Rappoport</name><affiliation><nlm:aff id="Aff32">Department of Biological Chemistry, Institute of Life Sciences, The Hebrew University of Jerusalem, Jerusalem, Israel</nlm:aff></affiliation><affiliation><nlm:aff id="Aff33">School of Computer Science and Engineering, The Hebrew University of Jerusalem, Jerusalem, Israel</nlm:aff></affiliation></author><author><name sortKey="Stern, Amos" sort="Stern, Amos" uniqKey="Stern A" first="Amos" last="Stern">Amos Stern</name><affiliation><nlm:aff id="Aff32">Department of Biological Chemistry, Institute of Life Sciences, The Hebrew University of Jerusalem, Jerusalem, Israel</nlm:aff></affiliation><affiliation><nlm:aff id="Aff33">School of Computer Science and Engineering, The Hebrew University of Jerusalem, Jerusalem, Israel</nlm:aff></affiliation></author><author><name sortKey="Cibrian Uhalte, Elena" sort="Cibrian Uhalte, Elena" uniqKey="Cibrian Uhalte E" first="Elena" last="Cibrian-Uhalte">Elena Cibrian-Uhalte</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Denny, Paul" sort="Denny, Paul" uniqKey="Denny P" first="Paul" last="Denny">Paul Denny</name><affiliation><nlm:aff id="Aff35">Centre for Cardiovascular Genetics, Institute of Cardiovascular Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Foulger, Rebecca E" sort="Foulger, Rebecca E" uniqKey="Foulger R" first="Rebecca E." last="Foulger">Rebecca E. Foulger</name><affiliation><nlm:aff id="Aff35">Centre for Cardiovascular Genetics, Institute of Cardiovascular Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Hieta, Reija" sort="Hieta, Reija" uniqKey="Hieta R" first="Reija" last="Hieta">Reija Hieta</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Legge, Duncan" sort="Legge, Duncan" uniqKey="Legge D" first="Duncan" last="Legge">Duncan Legge</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Lovering, Ruth C" sort="Lovering, Ruth C" uniqKey="Lovering R" first="Ruth C." last="Lovering">Ruth C. Lovering</name><affiliation><nlm:aff id="Aff35">Centre for Cardiovascular Genetics, Institute of Cardiovascular Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Magrane, Michele" sort="Magrane, Michele" uniqKey="Magrane M" first="Michele" last="Magrane">Michele Magrane</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Melidoni, Anna N" sort="Melidoni, Anna N" uniqKey="Melidoni A" first="Anna N." last="Melidoni">Anna N. Melidoni</name><affiliation><nlm:aff id="Aff35">Centre for Cardiovascular Genetics, Institute of Cardiovascular Science, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Mutowo Meullenet, Prudence" sort="Mutowo Meullenet, Prudence" uniqKey="Mutowo Meullenet P" first="Prudence" last="Mutowo-Meullenet">Prudence Mutowo-Meullenet</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Pichler, Klemens" sort="Pichler, Klemens" uniqKey="Pichler K" first="Klemens" last="Pichler">Klemens Pichler</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Shypitsyna, Aleksandra" sort="Shypitsyna, Aleksandra" uniqKey="Shypitsyna A" first="Aleksandra" last="Shypitsyna">Aleksandra Shypitsyna</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Li, Biao" sort="Li, Biao" uniqKey="Li B" first="Biao" last="Li">Biao Li</name><affiliation><nlm:aff id="Aff2">Buck Institute for Research on Aging, Novato, CA USA</nlm:aff></affiliation></author><author><name sortKey="Zakeri, Pooya" sort="Zakeri, Pooya" uniqKey="Zakeri P" first="Pooya" last="Zakeri">Pooya Zakeri</name><affiliation><nlm:aff id="Aff36">Department of Electrical Engineering, STADIUS Center for Dynamical Systems, Signal Processing and Data Analytics, KU Leuven, Leuven, Belgium</nlm:aff></affiliation><affiliation><nlm:aff id="Aff37">iMinds Department Medical Information Technologies, Leuven, Belgium</nlm:aff></affiliation></author><author><name sortKey="Elshal, Sarah" sort="Elshal, Sarah" uniqKey="Elshal S" first="Sarah" last="Elshal">Sarah Elshal</name><affiliation><nlm:aff id="Aff36">Department of Electrical Engineering, STADIUS Center for Dynamical Systems, Signal Processing and Data Analytics, KU Leuven, Leuven, Belgium</nlm:aff></affiliation><affiliation><nlm:aff id="Aff37">iMinds Department Medical Information Technologies, Leuven, Belgium</nlm:aff></affiliation></author><author><name sortKey="Tranchevent, Leon Charles" sort="Tranchevent, Leon Charles" uniqKey="Tranchevent L" first="Léon-Charles" last="Tranchevent">Léon-Charles Tranchevent</name><affiliation><nlm:aff id="Aff38">Inserm UMR-S1052, CNRS UMR5286, Cancer Research Centre of Lyon, Lyon, France</nlm:aff></affiliation><affiliation><nlm:aff id="Aff39">Université de Lyon 1, Villeurbanne, France</nlm:aff></affiliation><affiliation><nlm:aff id="Aff40">Centre Léon Bérard, Lyon, France</nlm:aff></affiliation></author><author><name sortKey="Das, Sayoni" sort="Das, Sayoni" uniqKey="Das S" first="Sayoni" last="Das">Sayoni Das</name><affiliation><nlm:aff id="Aff41">Institute of Structural and Molecular Biology, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Dawson, Natalie L" sort="Dawson, Natalie L" uniqKey="Dawson N" first="Natalie L." last="Dawson">Natalie L. Dawson</name><affiliation><nlm:aff id="Aff41">Institute of Structural and Molecular Biology, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Lee, David" sort="Lee, David" uniqKey="Lee D" first="David" last="Lee">David Lee</name><affiliation><nlm:aff id="Aff41">Institute of Structural and Molecular Biology, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Lees, Jonathan G" sort="Lees, Jonathan G" uniqKey="Lees J" first="Jonathan G." last="Lees">Jonathan G. Lees</name><affiliation><nlm:aff id="Aff41">Institute of Structural and Molecular Biology, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Sillitoe, Ian" sort="Sillitoe, Ian" uniqKey="Sillitoe I" first="Ian" last="Sillitoe">Ian Sillitoe</name><affiliation><nlm:aff id="Aff41">Institute of Structural and Molecular Biology, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Bhat, Prajwal" sort="Bhat, Prajwal" uniqKey="Bhat P" first="Prajwal" last="Bhat">Prajwal Bhat</name><affiliation><nlm:aff id="Aff42">Cerenode Inc., Boston, MA USA</nlm:aff></affiliation></author><author><name sortKey="Nepusz, Tamas" sort="Nepusz, Tamas" uniqKey="Nepusz T" first="Tamás" last="Nepusz">Tamás Nepusz</name><affiliation><nlm:aff id="Aff43">Molde University College, Molde, Norway</nlm:aff></affiliation></author><author><name sortKey="Romero, Alfonso E" sort="Romero, Alfonso E" uniqKey="Romero A" first="Alfonso E." last="Romero">Alfonso E. Romero</name><affiliation><nlm:aff id="Aff44">Department of Computer Science, Centre for Systems and Synthetic Biology, Royal Holloway University of London, Egham, UK</nlm:aff></affiliation></author><author><name sortKey="Sasidharan, Rajkumar" sort="Sasidharan, Rajkumar" uniqKey="Sasidharan R" first="Rajkumar" last="Sasidharan">Rajkumar Sasidharan</name><affiliation><nlm:aff id="Aff45">Department of Molecular, Cell and Developmental Biology, University of California at Los Angeles, Los Angeles, CA USA</nlm:aff></affiliation></author><author><name sortKey="Yang, Haixuan" sort="Yang, Haixuan" uniqKey="Yang H" first="Haixuan" last="Yang">Haixuan Yang</name><affiliation><nlm:aff id="Aff46">School of Mathematics, Statistics and Applied Mathematics, National University of Ireland, Galway, Ireland</nlm:aff></affiliation></author><author><name sortKey="Paccanaro, Alberto" sort="Paccanaro, Alberto" uniqKey="Paccanaro A" first="Alberto" last="Paccanaro">Alberto Paccanaro</name><affiliation><nlm:aff id="Aff44">Department of Computer Science, Centre for Systems and Synthetic Biology, Royal Holloway University of London, Egham, UK</nlm:aff></affiliation></author><author><name sortKey="Gillis, Jesse" sort="Gillis, Jesse" uniqKey="Gillis J" first="Jesse" last="Gillis">Jesse Gillis</name><affiliation><nlm:aff id="Aff47">Stanley Institute for Cognitive Genomics Cold Spring Harbor Laboratory, New York, NY USA</nlm:aff></affiliation></author><author><name sortKey="Sede O Cortes, Adriana E" sort="Sede O Cortes, Adriana E" uniqKey="Sede O Cortes A" first="Adriana E." last="Sede O-Cortés">Adriana E. Sede O-Cortés</name><affiliation><nlm:aff id="Aff48">Graduate Program in Bioinformatics, University of British Columbia, Vancouver, Canada</nlm:aff></affiliation></author><author><name sortKey="Pavlidis, Paul" sort="Pavlidis, Paul" uniqKey="Pavlidis P" first="Paul" last="Pavlidis">Paul Pavlidis</name><affiliation><nlm:aff id="Aff49">Department of Psychiatry and Michael Smith Laboratories, University of British Columbia, Vancouver, Canada</nlm:aff></affiliation></author><author><name sortKey="Feng, Shou" sort="Feng, Shou" uniqKey="Feng S" first="Shou" last="Feng">Shou Feng</name><affiliation><nlm:aff id="Aff1">Department of Computer Science and Informatics, Indiana University, Bloomington, IN USA</nlm:aff></affiliation></author><author><name sortKey="Cejuela, Juan M" sort="Cejuela, Juan M" uniqKey="Cejuela J" first="Juan M." last="Cejuela">Juan M. Cejuela</name><affiliation><nlm:aff id="Aff50">Department for Bioinformatics and Computational Biology-I12, Technische Universität München, Garching, Germany</nlm:aff></affiliation></author><author><name sortKey="Goldberg, Tatyana" sort="Goldberg, Tatyana" uniqKey="Goldberg T" first="Tatyana" last="Goldberg">Tatyana Goldberg</name><affiliation><nlm:aff id="Aff50">Department for Bioinformatics and Computational Biology-I12, Technische Universität München, Garching, Germany</nlm:aff></affiliation></author><author><name sortKey="Hamp, Tobias" sort="Hamp, Tobias" uniqKey="Hamp T" first="Tobias" last="Hamp">Tobias Hamp</name><affiliation><nlm:aff id="Aff50">Department for Bioinformatics and Computational Biology-I12, Technische Universität München, Garching, Germany</nlm:aff></affiliation></author><author><name sortKey="Richter, Lothar" sort="Richter, Lothar" uniqKey="Richter L" first="Lothar" last="Richter">Lothar Richter</name><affiliation><nlm:aff id="Aff50">Department for Bioinformatics and Computational Biology-I12, Technische Universität München, Garching, Germany</nlm:aff></affiliation></author><author><name sortKey="Salamov, Asaf" sort="Salamov, Asaf" uniqKey="Salamov A" first="Asaf" last="Salamov">Asaf Salamov</name><affiliation><nlm:aff id="Aff51">DOE Joint Genome Institute, Walnut Creek, CA USA</nlm:aff></affiliation></author><author><name sortKey="Gabaldon, Toni" sort="Gabaldon, Toni" uniqKey="Gabaldon T" first="Toni" last="Gabaldon">Toni Gabaldon</name><affiliation><nlm:aff id="Aff52">Bioinformatics and Genomics, Centre for Genomic Regulation, Barcelona, Spain</nlm:aff></affiliation><affiliation><nlm:aff id="Aff53">Universitat Pompeu Fabra, Barcelona, Spain</nlm:aff></affiliation><affiliation><nlm:aff id="Aff54">Institució Catalana de Recerca i Estudis Avançats, Barcelona, Spain</nlm:aff></affiliation></author><author><name sortKey="Marcet Houben, Marina" sort="Marcet Houben, Marina" uniqKey="Marcet Houben M" first="Marina" last="Marcet-Houben">Marina Marcet-Houben</name><affiliation><nlm:aff id="Aff52">Bioinformatics and Genomics, Centre for Genomic Regulation, Barcelona, Spain</nlm:aff></affiliation><affiliation><nlm:aff id="Aff53">Universitat Pompeu Fabra, Barcelona, Spain</nlm:aff></affiliation></author><author><name sortKey="Supek, Fran" sort="Supek, Fran" uniqKey="Supek F" first="Fran" last="Supek">Fran Supek</name><affiliation><nlm:aff id="Aff53">Universitat Pompeu Fabra, Barcelona, Spain</nlm:aff></affiliation><affiliation><nlm:aff id="Aff55">Division of Electronics, Rudjer Boskovic Institute, Zagreb, Croatia</nlm:aff></affiliation><affiliation><nlm:aff id="Aff56">EMBL/CRG Systems Biology Research Unit, Centre for Genomic Regulation, Barcelona, Spain</nlm:aff></affiliation></author><author><name sortKey="Gong, Qingtian" sort="Gong, Qingtian" uniqKey="Gong Q" first="Qingtian" last="Gong">Qingtian Gong</name><affiliation><nlm:aff id="Aff57">State Key Laboratory of Genetic Engineering, Collaborative Innovation Center of Genetics and Development, Department of Biostatistics and Computational Biology, School of Life Science, Fudan University, Shanghai, China</nlm:aff></affiliation><affiliation><nlm:aff id="Aff58">Children’s Hospital of Fudan University, Shanghai, China</nlm:aff></affiliation></author><author><name sortKey="Ning, Wei" sort="Ning, Wei" uniqKey="Ning W" first="Wei" last="Ning">Wei Ning</name><affiliation><nlm:aff id="Aff57">State Key Laboratory of Genetic Engineering, Collaborative Innovation Center of Genetics and Development, Department of Biostatistics and Computational Biology, School of Life Science, Fudan University, Shanghai, China</nlm:aff></affiliation><affiliation><nlm:aff id="Aff58">Children’s Hospital of Fudan University, Shanghai, China</nlm:aff></affiliation></author><author><name sortKey="Zhou, Yuanpeng" sort="Zhou, Yuanpeng" uniqKey="Zhou Y" first="Yuanpeng" last="Zhou">Yuanpeng Zhou</name><affiliation><nlm:aff id="Aff57">State Key Laboratory of Genetic Engineering, Collaborative Innovation Center of Genetics and Development, Department of Biostatistics and Computational Biology, School of Life Science, Fudan University, Shanghai, China</nlm:aff></affiliation><affiliation><nlm:aff id="Aff58">Children’s Hospital of Fudan University, Shanghai, China</nlm:aff></affiliation></author><author><name sortKey="Tian, Weidong" sort="Tian, Weidong" uniqKey="Tian W" first="Weidong" last="Tian">Weidong Tian</name><affiliation><nlm:aff id="Aff57">State Key Laboratory of Genetic Engineering, Collaborative Innovation Center of Genetics and Development, Department of Biostatistics and Computational Biology, School of Life Science, Fudan University, Shanghai, China</nlm:aff></affiliation><affiliation><nlm:aff id="Aff58">Children’s Hospital of Fudan University, Shanghai, China</nlm:aff></affiliation></author><author><name sortKey="Falda, Marco" sort="Falda, Marco" uniqKey="Falda M" first="Marco" last="Falda">Marco Falda</name><affiliation><nlm:aff id="Aff59">Department of Molecular Medicine, University of Padua, Padua, Italy</nlm:aff></affiliation></author><author><name sortKey="Fontana, Paolo" sort="Fontana, Paolo" uniqKey="Fontana P" first="Paolo" last="Fontana">Paolo Fontana</name><affiliation><nlm:aff id="Aff60">Research and Innovation Center, Edmund Mach Foundation, San Michele all’Adige, Italy</nlm:aff></affiliation></author><author><name sortKey="Lavezzo, Enrico" sort="Lavezzo, Enrico" uniqKey="Lavezzo E" first="Enrico" last="Lavezzo">Enrico Lavezzo</name><affiliation><nlm:aff id="Aff59">Department of Molecular Medicine, University of Padua, Padua, Italy</nlm:aff></affiliation></author><author><name sortKey="Toppo, Stefano" sort="Toppo, Stefano" uniqKey="Toppo S" first="Stefano" last="Toppo">Stefano Toppo</name><affiliation><nlm:aff id="Aff59">Department of Molecular Medicine, University of Padua, Padua, Italy</nlm:aff></affiliation></author><author><name sortKey="Ferrari, Carlo" sort="Ferrari, Carlo" uniqKey="Ferrari C" first="Carlo" last="Ferrari">Carlo Ferrari</name><affiliation><nlm:aff id="Aff61">Department of Information Engineering, University of Padua, Padova, Italy</nlm:aff></affiliation></author><author><name sortKey="Giollo, Manuel" sort="Giollo, Manuel" uniqKey="Giollo M" first="Manuel" last="Giollo">Manuel Giollo</name><affiliation><nlm:aff id="Aff61">Department of Information Engineering, University of Padua, Padova, Italy</nlm:aff></affiliation><affiliation><nlm:aff id="Aff84">Department of Biomedical Sciences, University of Padua, Padova, Italy</nlm:aff></affiliation></author><author><name sortKey="Piovesan, Damiano" sort="Piovesan, Damiano" uniqKey="Piovesan D" first="Damiano" last="Piovesan">Damiano Piovesan</name><affiliation><nlm:aff id="Aff61">Department of Information Engineering, University of Padua, Padova, Italy</nlm:aff></affiliation></author><author><name sortKey="Tosatto, Silvio C E" sort="Tosatto, Silvio C E" uniqKey="Tosatto S" first="Silvio C. E." last="Tosatto">Silvio C. E. Tosatto</name><affiliation><nlm:aff id="Aff61">Department of Information Engineering, University of Padua, Padova, Italy</nlm:aff></affiliation></author><author><name sortKey="Del Pozo, Angela" sort="Del Pozo, Angela" uniqKey="Del Pozo A" first="Angela" last="Del Pozo">Angela Del Pozo</name><affiliation><nlm:aff id="Aff62">Instituto De Genetica Medica y Molecular, Hospital Universitario de La Paz, Madrid, Spain</nlm:aff></affiliation></author><author><name sortKey="Fernandez, Jose M" sort="Fernandez, Jose M" uniqKey="Fernandez J" first="José M." last="Fernández">José M. Fernández</name><affiliation><nlm:aff id="Aff63">Spanish National Bioinformatics Institute, Spanish National Cancer Research Institute, Madrid, Spain</nlm:aff></affiliation></author><author><name sortKey="Maietta, Paolo" sort="Maietta, Paolo" uniqKey="Maietta P" first="Paolo" last="Maietta">Paolo Maietta</name><affiliation><nlm:aff id="Aff64">Structural and Computational Biology Programme, Spanish National Cancer Research Institute, Madrid, Spain</nlm:aff></affiliation></author><author><name sortKey="Valencia, Alfonso" sort="Valencia, Alfonso" uniqKey="Valencia A" first="Alfonso" last="Valencia">Alfonso Valencia</name><affiliation><nlm:aff id="Aff64">Structural and Computational Biology Programme, Spanish National Cancer Research Institute, Madrid, Spain</nlm:aff></affiliation></author><author><name sortKey="Tress, Michael L" sort="Tress, Michael L" uniqKey="Tress M" first="Michael L." last="Tress">Michael L. Tress</name><affiliation><nlm:aff id="Aff64">Structural and Computational Biology Programme, Spanish National Cancer Research Institute, Madrid, Spain</nlm:aff></affiliation></author><author><name sortKey="Benso, Alfredo" sort="Benso, Alfredo" uniqKey="Benso A" first="Alfredo" last="Benso">Alfredo Benso</name><affiliation><nlm:aff id="Aff65">Control and Computer Engineering Department, Politecnico di Torino, Torino, Italy</nlm:aff></affiliation></author><author><name sortKey="Di Carlo, Stefano" sort="Di Carlo, Stefano" uniqKey="Di Carlo S" first="Stefano" last="Di Carlo">Stefano Di Carlo</name><affiliation><nlm:aff id="Aff65">Control and Computer Engineering Department, Politecnico di Torino, Torino, Italy</nlm:aff></affiliation></author><author><name sortKey="Politano, Gianfranco" sort="Politano, Gianfranco" uniqKey="Politano G" first="Gianfranco" last="Politano">Gianfranco Politano</name><affiliation><nlm:aff id="Aff65">Control and Computer Engineering Department, Politecnico di Torino, Torino, Italy</nlm:aff></affiliation></author><author><name sortKey="Savino, Alessandro" sort="Savino, Alessandro" uniqKey="Savino A" first="Alessandro" last="Savino">Alessandro Savino</name><affiliation><nlm:aff id="Aff65">Control and Computer Engineering Department, Politecnico di Torino, Torino, Italy</nlm:aff></affiliation></author><author><name sortKey="Rehman, Hafeez Ur" sort="Rehman, Hafeez Ur" uniqKey="Rehman H" first="Hafeez Ur" last="Rehman">Hafeez Ur Rehman</name><affiliation><nlm:aff id="Aff66">National University of Computer & Emerging Sciences, Islamabad, Pakistan</nlm:aff></affiliation></author><author><name sortKey="Re, Matteo" sort="Re, Matteo" uniqKey="Re M" first="Matteo" last="Re">Matteo Re</name><affiliation><nlm:aff id="Aff67">Anacleto Lab, Dipartimento di informatica, Università degli Studi di Milano, Milan, Italy</nlm:aff></affiliation></author><author><name sortKey="Mesiti, Marco" sort="Mesiti, Marco" uniqKey="Mesiti M" first="Marco" last="Mesiti">Marco Mesiti</name><affiliation><nlm:aff id="Aff67">Anacleto Lab, Dipartimento di informatica, Università degli Studi di Milano, Milan, Italy</nlm:aff></affiliation></author><author><name sortKey="Valentini, Giorgio" sort="Valentini, Giorgio" uniqKey="Valentini G" first="Giorgio" last="Valentini">Giorgio Valentini</name><affiliation><nlm:aff id="Aff67">Anacleto Lab, Dipartimento di informatica, Università degli Studi di Milano, Milan, Italy</nlm:aff></affiliation></author><author><name sortKey="Bargsten, Joachim W" sort="Bargsten, Joachim W" uniqKey="Bargsten J" first="Joachim W." last="Bargsten">Joachim W. Bargsten</name><affiliation><nlm:aff id="Aff68">Applied Bioinformatics, Bioscience, Wageningen University and Research Centre, Wageningen, Netherlands</nlm:aff></affiliation></author><author><name sortKey="Van Dijk, Aalt D J" sort="Van Dijk, Aalt D J" uniqKey="Van Dijk A" first="Aalt D. J." last="Van Dijk">Aalt D. J. Van Dijk</name><affiliation><nlm:aff id="Aff68">Applied Bioinformatics, Bioscience, Wageningen University and Research Centre, Wageningen, Netherlands</nlm:aff></affiliation><affiliation><nlm:aff id="Aff69">Biometris, Wageningen University, Wageningen, Netherlands</nlm:aff></affiliation></author><author><name sortKey="Gemovic, Branislava" sort="Gemovic, Branislava" uniqKey="Gemovic B" first="Branislava" last="Gemovic">Branislava Gemovic</name><affiliation><nlm:aff id="Aff70">Center for Multidisciplinary Research, Institute of Nuclear Sciences Vinca, University of Belgrade, Belgrade, Serbia</nlm:aff></affiliation></author><author><name sortKey="Glisic, Sanja" sort="Glisic, Sanja" uniqKey="Glisic S" first="Sanja" last="Glisic">Sanja Glisic</name><affiliation><nlm:aff id="Aff70">Center for Multidisciplinary Research, Institute of Nuclear Sciences Vinca, University of Belgrade, Belgrade, Serbia</nlm:aff></affiliation></author><author><name sortKey="Perovic, Vladmir" sort="Perovic, Vladmir" uniqKey="Perovic V" first="Vladmir" last="Perovic">Vladmir Perovic</name><affiliation><nlm:aff id="Aff70">Center for Multidisciplinary Research, Institute of Nuclear Sciences Vinca, University of Belgrade, Belgrade, Serbia</nlm:aff></affiliation></author><author><name sortKey="Veljkovic, Veljko" sort="Veljkovic, Veljko" uniqKey="Veljkovic V" first="Veljko" last="Veljkovic">Veljko Veljkovic</name><affiliation><nlm:aff id="Aff70">Center for Multidisciplinary Research, Institute of Nuclear Sciences Vinca, University of Belgrade, Belgrade, Serbia</nlm:aff></affiliation></author><author><name sortKey="Veljkovic, Nevena" sort="Veljkovic, Nevena" uniqKey="Veljkovic N" first="Nevena" last="Veljkovic">Nevena Veljkovic</name><affiliation><nlm:aff id="Aff70">Center for Multidisciplinary Research, Institute of Nuclear Sciences Vinca, University of Belgrade, Belgrade, Serbia</nlm:aff></affiliation></author><author><name sortKey="Almeida E Silva, Danillo C" sort="Almeida E Silva, Danillo C" uniqKey="Almeida E Silva D" first="Danillo C." last="Almeida-E-Silva">Danillo C. Almeida-E-Silva</name><affiliation><nlm:aff id="Aff71">Department of Computing and Mathematics FFCLRP-USP, University of Sao Paulo, Ribeirao Preto, Brazil</nlm:aff></affiliation></author><author><name sortKey="Vencio, Ricardo Z N" sort="Vencio, Ricardo Z N" uniqKey="Vencio R" first="Ricardo Z. N." last="Vencio">Ricardo Z. N. Vencio</name><affiliation><nlm:aff id="Aff71">Department of Computing and Mathematics FFCLRP-USP, University of Sao Paulo, Ribeirao Preto, Brazil</nlm:aff></affiliation></author><author><name sortKey="Sharan, Malvika" sort="Sharan, Malvika" uniqKey="Sharan M" first="Malvika" last="Sharan">Malvika Sharan</name><affiliation><nlm:aff id="Aff72">Institute for Molecular Infection Biology, University of Würzburg, Würzburg, Germany</nlm:aff></affiliation></author><author><name sortKey="Vogel, Jorg" sort="Vogel, Jorg" uniqKey="Vogel J" first="Jörg" last="Vogel">Jörg Vogel</name><affiliation><nlm:aff id="Aff72">Institute for Molecular Infection Biology, University of Würzburg, Würzburg, Germany</nlm:aff></affiliation></author><author><name sortKey="Kansakar, Lakesh" sort="Kansakar, Lakesh" uniqKey="Kansakar L" first="Lakesh" last="Kansakar">Lakesh Kansakar</name><affiliation><nlm:aff id="Aff73">Department of Computer and Information Sciences, Temple University, Philadelphia, PA USA</nlm:aff></affiliation></author><author><name sortKey="Zhang, Shanshan" sort="Zhang, Shanshan" uniqKey="Zhang S" first="Shanshan" last="Zhang">Shanshan Zhang</name><affiliation><nlm:aff id="Aff73">Department of Computer and Information Sciences, Temple University, Philadelphia, PA USA</nlm:aff></affiliation></author><author><name sortKey="Vucetic, Slobodan" sort="Vucetic, Slobodan" uniqKey="Vucetic S" first="Slobodan" last="Vucetic">Slobodan Vucetic</name><affiliation><nlm:aff id="Aff73">Department of Computer and Information Sciences, Temple University, Philadelphia, PA USA</nlm:aff></affiliation></author><author><name sortKey="Wang, Zheng" sort="Wang, Zheng" uniqKey="Wang Z" first="Zheng" last="Wang">Zheng Wang</name><affiliation><nlm:aff id="Aff74">University of Southern Mississippi, Hattiesburg, MS USA</nlm:aff></affiliation></author><author><name sortKey="Sternberg, Michael J E" sort="Sternberg, Michael J E" uniqKey="Sternberg M" first="Michael J. E." last="Sternberg">Michael J. E. Sternberg</name><affiliation><nlm:aff id="Aff34">Centre for Integrative Systems Biology and Bioinformatics, Department of Life Sciences, Imperial College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Wass, Mark N" sort="Wass, Mark N" uniqKey="Wass M" first="Mark N." last="Wass">Mark N. Wass</name><affiliation><nlm:aff id="Aff75">School of Biosciences, University of Kent, Canterbury, Kent, UK</nlm:aff></affiliation></author><author><name sortKey="Huntley, Rachael P" sort="Huntley, Rachael P" uniqKey="Huntley R" first="Rachael P." last="Huntley">Rachael P. Huntley</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Martin, Maria J" sort="Martin, Maria J" uniqKey="Martin M" first="Maria J." last="Martin">Maria J. Martin</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="O Onovan, Claire" sort="O Onovan, Claire" uniqKey="O Onovan C" first="Claire" last="O Onovan">Claire O Onovan</name><affiliation><nlm:aff id="Aff23">European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</nlm:aff></affiliation></author><author><name sortKey="Robinson, Peter N" sort="Robinson, Peter N" uniqKey="Robinson P" first="Peter N." last="Robinson">Peter N. Robinson</name><affiliation><nlm:aff id="Aff76">Institut für Medizinische Genetik und Humangenetik, Charité - Universitätsmedizin Berlin, Berlin, Germany</nlm:aff></affiliation></author><author><name sortKey="Moreau, Yves" sort="Moreau, Yves" uniqKey="Moreau Y" first="Yves" last="Moreau">Yves Moreau</name><affiliation><nlm:aff id="Aff77">Department of Electrical Engineering ESAT-SCD and IBBT-KU Leuven Future Health Department, Katholieke Universiteit Leuven, Leuven, Belgium</nlm:aff></affiliation></author><author><name sortKey="Tramontano, Anna" sort="Tramontano, Anna" uniqKey="Tramontano A" first="Anna" last="Tramontano">Anna Tramontano</name><affiliation><nlm:aff id="Aff5">University of Rome, La Sapienza, Rome, Italy</nlm:aff></affiliation></author><author><name sortKey="Babbitt, Patricia C" sort="Babbitt, Patricia C" uniqKey="Babbitt P" first="Patricia C." last="Babbitt">Patricia C. Babbitt</name><affiliation><nlm:aff id="Aff78">California Institute for Quantitative Biosciences, University of California San Francisco, San Francisco, CA USA</nlm:aff></affiliation></author><author><name sortKey="Brenner, Steven E" sort="Brenner, Steven E" uniqKey="Brenner S" first="Steven E." last="Brenner">Steven E. Brenner</name><affiliation><nlm:aff id="Aff17">Department of Plant and Microbial Biology, University of California Berkeley, Berkeley, CA USA</nlm:aff></affiliation></author><author><name sortKey="Linial, Michal" sort="Linial, Michal" uniqKey="Linial M" first="Michal" last="Linial">Michal Linial</name><affiliation><nlm:aff id="Aff79">Department of Chemical Biology, The Hebrew University of Jerusalem, Jerusalem, Israel</nlm:aff></affiliation></author><author><name sortKey="Orengo, Christine A" sort="Orengo, Christine A" uniqKey="Orengo C" first="Christine A." last="Orengo">Christine A. Orengo</name><affiliation><nlm:aff id="Aff41">Institute of Structural and Molecular Biology, University College London, London, UK</nlm:aff></affiliation></author><author><name sortKey="Rost, Burkhard" sort="Rost, Burkhard" uniqKey="Rost B" first="Burkhard" last="Rost">Burkhard Rost</name><affiliation><nlm:aff id="Aff50">Department for Bioinformatics and Computational Biology-I12, Technische Universität München, Garching, Germany</nlm:aff></affiliation></author><author><name sortKey="Greene, Casey S" sort="Greene, Casey S" uniqKey="Greene C" first="Casey S." last="Greene">Casey S. Greene</name><affiliation><nlm:aff id="Aff80">Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania, Philadelphia, PA USA</nlm:aff></affiliation></author><author><name sortKey="Mooney, Sean D" sort="Mooney, Sean D" uniqKey="Mooney S" first="Sean D." last="Mooney">Sean D. Mooney</name><affiliation><nlm:aff id="Aff81">Department of Biomedical Informatics and Medical Education, University of Washington, Seattle, WA USA</nlm:aff></affiliation></author><author><name sortKey="Friedberg, Iddo" sort="Friedberg, Iddo" uniqKey="Friedberg I" first="Iddo" last="Friedberg">Iddo Friedberg</name><affiliation><nlm:aff id="Aff4">Department of Microbiology, Miami University, Oxford, OH USA</nlm:aff></affiliation><affiliation><nlm:aff id="Aff82">Department of Computer Science, Miami University, Oxford, OH USA</nlm:aff></affiliation></author><author><name sortKey="Radivojac, Predrag" sort="Radivojac, Predrag" uniqKey="Radivojac P" first="Predrag" last="Radivojac">Predrag Radivojac</name><affiliation><nlm:aff id="Aff1">Department of Computer Science and Informatics, Indiana University, Bloomington, IN USA</nlm:aff></affiliation></author></analytic><series><title level="j">Genome Biology</title><idno type="ISSN">1474-7596</idno><idno type="eISSN">1474-760X</idno><imprint><date when="2016">2016</date></imprint></series></biblStruct></sourceDesc></fileDesc><profileDesc><textClass></textClass></profileDesc></teiHeader><front><div type="abstract" xml:lang="en"><sec><title>Background</title><p>A major bottleneck in our understanding of the molecular underpinnings of life is the assignment of function to proteins. While molecular experiments provide the most reliable annotation of proteins, their relatively low throughput and restricted purview have led to an increasing role for computational function prediction. However, assessing methods for protein function prediction and tracking progress in the field remain challenging.</p></sec><sec><title>Results</title><p>We conducted the second critical assessment of functional annotation (CAFA), a timed challenge to assess computational methods that automatically assign protein function. We evaluated 126 methods from 56 research groups for their ability to predict biological functions using Gene Ontology and gene-disease associations using Human Phenotype Ontology on a set of 3681 proteins from 18 species. CAFA2 featured expanded analysis compared with CAFA1, with regards to data set size, variety, and assessment metrics. To review progress in the field, the analysis compared the best methods from CAFA1 to those of CAFA2.</p></sec><sec><title>Conclusions</title><p>The top-performing methods in CAFA2 outperformed those from CAFA1. This increased accuracy can be attributed to a combination of the growing number of experimental annotations and improved methods for function prediction. The assessment also revealed that the definition of top-performing algorithms is ontology specific, that different performance metrics can be used to probe the nature of accurate predictions, and the relative diversity of predictions in the biological process and human phenotype ontologies. While there was methodological improvement between CAFA1 and CAFA2, the interpretation of results and usefulness of individual methods remain context-dependent.</p></sec><sec><title>Electronic supplementary material</title><p>The online version of this article (doi:10.1186/s13059-016-1037-6) contains supplementary material, which is available to authorized users.</p></sec></div></front><back><div1 type="bibliography"><listBibl><biblStruct><analytic><author><name sortKey="Costello, Jc" uniqKey="Costello J">JC Costello</name></author><author><name sortKey="Stolovitzky, G" uniqKey="Stolovitzky G">G Stolovitzky</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Radivojac, P" uniqKey="Radivojac P">P Radivojac</name></author><author><name sortKey="Clark, Wt" uniqKey="Clark W">WT Clark</name></author><author><name sortKey="Oron, Tr" uniqKey="Oron T">TR Oron</name></author><author><name sortKey="Schnoes, Am" uniqKey="Schnoes A">AM Schnoes</name></author><author><name sortKey="Wittkop, T" uniqKey="Wittkop T">T Wittkop</name></author><author><name sortKey="Sokolov, A" uniqKey="Sokolov A">A Sokolov</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Ashburner, M" uniqKey="Ashburner M">M Ashburner</name></author><author><name sortKey="Ball, Ca" uniqKey="Ball C">CA Ball</name></author><author><name sortKey="Blake, Ja" uniqKey="Blake J">JA Blake</name></author><author><name sortKey="Botstein, D" uniqKey="Botstein D">D Botstein</name></author><author><name sortKey="Butler, H" uniqKey="Butler H">H Butler</name></author><author><name sortKey="Cherry, Jm" uniqKey="Cherry J">JM Cherry</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Dessimoz, C" uniqKey="Dessimoz C">C Dessimoz</name></author><author><name sortKey="Skunca, N" uniqKey="Skunca N">N Skunca</name></author><author><name sortKey="Thomas, Pd" uniqKey="Thomas P">PD Thomas</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Gillis, J" uniqKey="Gillis J">J Gillis</name></author><author><name sortKey="Pavlidis, P" uniqKey="Pavlidis P">P Pavlidis</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Schnoes, Am" uniqKey="Schnoes A">AM Schnoes</name></author><author><name sortKey="Ream, Dc" uniqKey="Ream D">DC Ream</name></author><author><name sortKey="Thorman, Aw" uniqKey="Thorman A">AW Thorman</name></author><author><name sortKey="Babbitt, Pc" uniqKey="Babbitt P">PC Babbitt</name></author><author><name sortKey="Friedberg, I" uniqKey="Friedberg I">I Friedberg</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Jiang, Y" uniqKey="Jiang Y">Y Jiang</name></author><author><name sortKey="Clark, Wt" uniqKey="Clark W">WT Clark</name></author><author><name sortKey="Friedberg, I" uniqKey="Friedberg I">I Friedberg</name></author><author><name sortKey="Radivojac, P" uniqKey="Radivojac P">P Radivojac</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Robinson, Pn" uniqKey="Robinson P">PN Robinson</name></author><author><name sortKey="Mundlos, S" uniqKey="Mundlos S">S Mundlos</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Moreau, Y" uniqKey="Moreau Y">Y Moreau</name></author><author><name sortKey="Tranchevent, Lc" uniqKey="Tranchevent L">LC Tranchevent</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Clark, Wt" uniqKey="Clark W">WT Clark</name></author><author><name sortKey="Radivojac, P" uniqKey="Radivojac P">P Radivojac</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Bairoch, A" uniqKey="Bairoch A">A Bairoch</name></author><author><name sortKey="Apweiler, R" uniqKey="Apweiler R">R Apweiler</name></author><author><name sortKey="Wu, Ch" uniqKey="Wu C">CH Wu</name></author><author><name sortKey="Barker, Wc" uniqKey="Barker W">WC Barker</name></author><author><name sortKey="Boeckmann, B" uniqKey="Boeckmann B">B Boeckmann</name></author><author><name sortKey="Ferro, S" uniqKey="Ferro S">S Ferro</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Huntley, Rp" uniqKey="Huntley R">RP Huntley</name></author><author><name sortKey="Sawford, T" uniqKey="Sawford T">T Sawford</name></author><author><name sortKey="Mutowo Meullenet, P" uniqKey="Mutowo Meullenet P">P Mutowo-Meullenet</name></author><author><name sortKey="Shypitsyna, A" uniqKey="Shypitsyna A">A Shypitsyna</name></author><author><name sortKey="Bonilla, C" uniqKey="Bonilla C">C Bonilla</name></author><author><name sortKey="Martin, Mj" uniqKey="Martin M">MJ Martin</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Clark, Wt" uniqKey="Clark W">WT Clark</name></author><author><name sortKey="Radivojac, P" uniqKey="Radivojac P">P Radivojac</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Altschul, Sf" uniqKey="Altschul S">SF Altschul</name></author><author><name sortKey="Madden, Tl" uniqKey="Madden T">TL Madden</name></author><author><name sortKey="Schaffer, Aa" uniqKey="Schaffer A">AA Schaffer</name></author><author><name sortKey="Zhang, J" uniqKey="Zhang J">J Zhang</name></author><author><name sortKey="Zhang, Z" uniqKey="Zhang Z">Z Zhang</name></author><author><name sortKey="Miller, W" uniqKey="Miller W">W Miller</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Efron, B" uniqKey="Efron B">B Efron</name></author><author><name sortKey="Tibshirani, Rj" uniqKey="Tibshirani R">RJ Tibshirani</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Cal, S" uniqKey="Cal S">S Cal</name></author><author><name sortKey="Arguelles, Jm" uniqKey="Arguelles J">JM Argüelles</name></author><author><name sortKey="Fernandez, Pl" uniqKey="Fernandez P">PL Fernández</name></author><author><name sortKey="L Pez Ot N, C" uniqKey="L Pez Ot N C">C López-Otın</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Wolfsberg, Tg" uniqKey="Wolfsberg T">TG Wolfsberg</name></author><author><name sortKey="Straight, Pd" uniqKey="Straight P">PD Straight</name></author><author><name sortKey="Gerena, Rl" uniqKey="Gerena R">RL Gerena</name></author><author><name sortKey="Huovila, A Pj" uniqKey="Huovila A">A-PJ Huovila</name></author><author><name sortKey="Primakoff, P" uniqKey="Primakoff P">P Primakoff</name></author><author><name sortKey="Myles, Dg" uniqKey="Myles D">DG Myles</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Brocker, Cn" uniqKey="Brocker C">CN Brocker</name></author><author><name sortKey="Vasiliou, V" uniqKey="Vasiliou V">V Vasiliou</name></author><author><name sortKey="Nebert, Dw" uniqKey="Nebert D">DW Nebert</name></author></analytic></biblStruct><biblStruct><analytic><author><name sortKey="Wass, Mn" uniqKey="Wass M">MN Wass</name></author><author><name sortKey="Mooney, Sd" uniqKey="Mooney S">SD Mooney</name></author><author><name sortKey="Linial, M" uniqKey="Linial M">M Linial</name></author><author><name sortKey="Radivojac, P" uniqKey="Radivojac P">P Radivojac</name></author><author><name sortKey="Friedberg, I" uniqKey="Friedberg I">I Friedberg</name></author></analytic></biblStruct></listBibl></div1></back></TEI><pmc article-type="research-article"><pmc-dir>properties open_access</pmc-dir>
<front><journal-meta><journal-id journal-id-type="nlm-ta">Genome Biol</journal-id><journal-id journal-id-type="iso-abbrev">Genome Biol</journal-id><journal-title-group><journal-title>Genome Biology</journal-title></journal-title-group><issn pub-type="ppub">1474-7596</issn><issn pub-type="epub">1474-760X</issn><publisher><publisher-name>BioMed Central</publisher-name><publisher-loc>London</publisher-loc></publisher></journal-meta><article-meta><article-id pub-id-type="pmid">27604469</article-id><article-id pub-id-type="pmc">5015320</article-id><article-id pub-id-type="publisher-id">1037</article-id><article-id pub-id-type="doi">10.1186/s13059-016-1037-6</article-id><article-categories><subj-group subj-group-type="heading"><subject>Research</subject></subj-group></article-categories><title-group><article-title>An expanded evaluation of protein function prediction methods shows an improvement in accuracy</article-title></title-group><contrib-group><contrib contrib-type="author"><name><surname>Jiang</surname><given-names>Yuxiang</given-names></name><xref ref-type="aff" rid="Aff1">1</xref></contrib><contrib contrib-type="author"><name><surname>Oron</surname><given-names>Tal Ronnen</given-names></name><xref ref-type="aff" rid="Aff2">2</xref></contrib><contrib contrib-type="author"><name><surname>Clark</surname><given-names>Wyatt T.</given-names></name><xref ref-type="aff" rid="Aff3">3</xref></contrib><contrib contrib-type="author"><name><surname>Bankapur</surname><given-names>Asma R.</given-names></name><xref ref-type="aff" rid="Aff4">4</xref></contrib><contrib contrib-type="author"><name><surname>D’Andrea</surname><given-names>Daniel</given-names></name><xref ref-type="aff" rid="Aff5">5</xref></contrib><contrib contrib-type="author"><name><surname>Lepore</surname><given-names>Rosalba</given-names></name><xref ref-type="aff" rid="Aff5">5</xref></contrib><contrib contrib-type="author"><name><surname>Funk</surname><given-names>Christopher S.</given-names></name><xref ref-type="aff" rid="Aff6">6</xref></contrib><contrib contrib-type="author"><name><surname>Kahanda</surname><given-names>Indika</given-names></name><xref ref-type="aff" rid="Aff7">7</xref></contrib><contrib contrib-type="author"><name><surname>Verspoor</surname><given-names>Karin M.</given-names></name><xref ref-type="aff" rid="Aff8">8</xref><xref ref-type="aff" rid="Aff9">9</xref></contrib><contrib contrib-type="author"><name><surname>Ben-Hur</surname><given-names>Asa</given-names></name><xref ref-type="aff" rid="Aff7">7</xref></contrib><contrib contrib-type="author"><name><surname>Koo</surname><given-names>Da Chen Emily</given-names></name><xref ref-type="aff" rid="Aff10">10</xref></contrib><contrib contrib-type="author"><name><surname>Penfold-Brown</surname><given-names>Duncan</given-names></name><xref ref-type="aff" rid="Aff11">11</xref><xref ref-type="aff" rid="Aff12">12</xref></contrib><contrib contrib-type="author"><name><surname>Shasha</surname><given-names>Dennis</given-names></name><xref ref-type="aff" rid="Aff13">13</xref></contrib><contrib contrib-type="author"><name><surname>Youngs</surname><given-names>Noah</given-names></name><xref ref-type="aff" rid="Aff12">12</xref><xref ref-type="aff" rid="Aff13">13</xref><xref ref-type="aff" rid="Aff14">14</xref></contrib><contrib contrib-type="author"><name><surname>Bonneau</surname><given-names>Richard</given-names></name><xref ref-type="aff" rid="Aff13">13</xref><xref ref-type="aff" rid="Aff14">14</xref><xref ref-type="aff" rid="Aff15">15</xref></contrib><contrib contrib-type="author"><name><surname>Lin</surname><given-names>Alexandra</given-names></name><xref ref-type="aff" rid="Aff16">16</xref></contrib><contrib contrib-type="author"><name><surname>Sahraeian</surname><given-names>Sayed M. E.</given-names></name><xref ref-type="aff" rid="Aff17">17</xref></contrib><contrib contrib-type="author"><name><surname>Martelli</surname><given-names>Pier Luigi</given-names></name><xref ref-type="aff" rid="Aff18">18</xref></contrib><contrib contrib-type="author"><name><surname>Profiti</surname><given-names>Giuseppe</given-names></name><xref ref-type="aff" rid="Aff18">18</xref></contrib><contrib contrib-type="author"><name><surname>Casadio</surname><given-names>Rita</given-names></name><xref ref-type="aff" rid="Aff18">18</xref></contrib><contrib contrib-type="author"><name><surname>Cao</surname><given-names>Renzhi</given-names></name><xref ref-type="aff" rid="Aff19">19</xref></contrib><contrib contrib-type="author"><name><surname>Zhong</surname><given-names>Zhaolong</given-names></name><xref ref-type="aff" rid="Aff19">19</xref></contrib><contrib contrib-type="author"><name><surname>Cheng</surname><given-names>Jianlin</given-names></name><xref ref-type="aff" rid="Aff19">19</xref></contrib><contrib contrib-type="author"><name><surname>Altenhoff</surname><given-names>Adrian</given-names></name><xref ref-type="aff" rid="Aff20">20</xref><xref ref-type="aff" rid="Aff21">21</xref></contrib><contrib contrib-type="author"><name><surname>Skunca</surname><given-names>Nives</given-names></name><xref ref-type="aff" rid="Aff20">20</xref><xref ref-type="aff" rid="Aff21">21</xref></contrib><contrib contrib-type="author"><name><surname>Dessimoz</surname><given-names>Christophe</given-names></name><xref ref-type="aff" rid="Aff22">22</xref><xref ref-type="aff" rid="Aff87">87</xref><xref ref-type="aff" rid="Aff88">88</xref></contrib><contrib contrib-type="author"><name><surname>Dogan</surname><given-names>Tunca</given-names></name><xref ref-type="aff" rid="Aff23">23</xref></contrib><contrib contrib-type="author"><name><surname>Hakala</surname><given-names>Kai</given-names></name><xref ref-type="aff" rid="Aff24">24</xref><xref ref-type="aff" rid="Aff25">25</xref></contrib><contrib contrib-type="author"><name><surname>Kaewphan</surname><given-names>Suwisa</given-names></name><xref ref-type="aff" rid="Aff24">24</xref><xref ref-type="aff" rid="Aff25">25</xref><xref ref-type="aff" rid="Aff26">26</xref></contrib><contrib contrib-type="author"><name><surname>Mehryary</surname><given-names>Farrokh</given-names></name><xref ref-type="aff" rid="Aff24">24</xref><xref ref-type="aff" rid="Aff25">25</xref></contrib><contrib contrib-type="author"><name><surname>Salakoski</surname><given-names>Tapio</given-names></name><xref ref-type="aff" rid="Aff24">24</xref><xref ref-type="aff" rid="Aff26">26</xref></contrib><contrib contrib-type="author"><name><surname>Ginter</surname><given-names>Filip</given-names></name><xref ref-type="aff" rid="Aff24">24</xref></contrib><contrib contrib-type="author"><name><surname>Fang</surname><given-names>Hai</given-names></name><xref ref-type="aff" rid="Aff27">27</xref></contrib><contrib contrib-type="author"><name><surname>Smithers</surname><given-names>Ben</given-names></name><xref ref-type="aff" rid="Aff27">27</xref></contrib><contrib contrib-type="author"><name><surname>Oates</surname><given-names>Matt</given-names></name><xref ref-type="aff" rid="Aff27">27</xref></contrib><contrib contrib-type="author"><name><surname>Gough</surname><given-names>Julian</given-names></name><xref ref-type="aff" rid="Aff27">27</xref></contrib><contrib contrib-type="author"><name><surname>Törönen</surname><given-names>Petri</given-names></name><xref ref-type="aff" rid="Aff28">28</xref></contrib><contrib contrib-type="author"><name><surname>Koskinen</surname><given-names>Patrik</given-names></name><xref ref-type="aff" rid="Aff28">28</xref></contrib><contrib contrib-type="author"><name><surname>Holm</surname><given-names>Liisa</given-names></name><xref ref-type="aff" rid="Aff28">28</xref><xref ref-type="aff" rid="Aff86">86</xref></contrib><contrib contrib-type="author"><name><surname>Chen</surname><given-names>Ching-Tai</given-names></name><xref ref-type="aff" rid="Aff29">29</xref></contrib><contrib contrib-type="author"><name><surname>Hsu</surname><given-names>Wen-Lian</given-names></name><xref ref-type="aff" rid="Aff29">29</xref></contrib><contrib contrib-type="author"><name><surname>Bryson</surname><given-names>Kevin</given-names></name><xref ref-type="aff" rid="Aff22">22</xref></contrib><contrib contrib-type="author"><name><surname>Cozzetto</surname><given-names>Domenico</given-names></name><xref ref-type="aff" rid="Aff22">22</xref></contrib><contrib contrib-type="author"><name><surname>Minneci</surname><given-names>Federico</given-names></name><xref ref-type="aff" rid="Aff22">22</xref></contrib><contrib contrib-type="author"><name><surname>Jones</surname><given-names>David T.</given-names></name><xref ref-type="aff" rid="Aff22">22</xref></contrib><contrib contrib-type="author"><name><surname>Chapman</surname><given-names>Samuel</given-names></name><xref ref-type="aff" rid="Aff30">30</xref></contrib><contrib contrib-type="author"><name><surname>BKC</surname><given-names>Dukka</given-names></name><xref ref-type="aff" rid="Aff30">30</xref></contrib><contrib contrib-type="author"><name><surname>Khan</surname><given-names>Ishita K.</given-names></name><xref ref-type="aff" rid="Aff31">31</xref></contrib><contrib contrib-type="author"><name><surname>Kihara</surname><given-names>Daisuke</given-names></name><xref ref-type="aff" rid="Aff31">31</xref><xref ref-type="aff" rid="Aff85">85</xref></contrib><contrib contrib-type="author"><name><surname>Ofer</surname><given-names>Dan</given-names></name><xref ref-type="aff" rid="Aff32">32</xref></contrib><contrib contrib-type="author"><name><surname>Rappoport</surname><given-names>Nadav</given-names></name><xref ref-type="aff" rid="Aff32">32</xref><xref ref-type="aff" rid="Aff33">33</xref></contrib><contrib contrib-type="author"><name><surname>Stern</surname><given-names>Amos</given-names></name><xref ref-type="aff" rid="Aff32">32</xref><xref ref-type="aff" rid="Aff33">33</xref></contrib><contrib contrib-type="author"><name><surname>Cibrian-Uhalte</surname><given-names>Elena</given-names></name><xref ref-type="aff" rid="Aff23">23</xref></contrib><contrib contrib-type="author"><name><surname>Denny</surname><given-names>Paul</given-names></name><xref ref-type="aff" rid="Aff35">35</xref></contrib><contrib contrib-type="author"><name><surname>Foulger</surname><given-names>Rebecca E.</given-names></name><xref ref-type="aff" rid="Aff35">35</xref></contrib><contrib contrib-type="author"><name><surname>Hieta</surname><given-names>Reija</given-names></name><xref ref-type="aff" rid="Aff23">23</xref></contrib><contrib contrib-type="author"><name><surname>Legge</surname><given-names>Duncan</given-names></name><xref ref-type="aff" rid="Aff23">23</xref></contrib><contrib contrib-type="author"><name><surname>Lovering</surname><given-names>Ruth C.</given-names></name><xref ref-type="aff" rid="Aff35">35</xref></contrib><contrib contrib-type="author"><name><surname>Magrane</surname><given-names>Michele</given-names></name><xref ref-type="aff" rid="Aff23">23</xref></contrib><contrib contrib-type="author"><name><surname>Melidoni</surname><given-names>Anna N.</given-names></name><xref ref-type="aff" rid="Aff35">35</xref></contrib><contrib contrib-type="author"><name><surname>Mutowo-Meullenet</surname><given-names>Prudence</given-names></name><xref ref-type="aff" rid="Aff23">23</xref></contrib><contrib contrib-type="author"><name><surname>Pichler</surname><given-names>Klemens</given-names></name><xref ref-type="aff" rid="Aff23">23</xref></contrib><contrib contrib-type="author"><name><surname>Shypitsyna</surname><given-names>Aleksandra</given-names></name><xref ref-type="aff" rid="Aff23">23</xref></contrib><contrib contrib-type="author"><name><surname>Li</surname><given-names>Biao</given-names></name><xref ref-type="aff" rid="Aff2">2</xref></contrib><contrib contrib-type="author"><name><surname>Zakeri</surname><given-names>Pooya</given-names></name><xref ref-type="aff" rid="Aff36">36</xref><xref ref-type="aff" rid="Aff37">37</xref></contrib><contrib contrib-type="author"><name><surname>ElShal</surname><given-names>Sarah</given-names></name><xref ref-type="aff" rid="Aff36">36</xref><xref ref-type="aff" rid="Aff37">37</xref></contrib><contrib contrib-type="author"><name><surname>Tranchevent</surname><given-names>Léon-Charles</given-names></name><xref ref-type="aff" rid="Aff38">38</xref><xref ref-type="aff" rid="Aff39">39</xref><xref ref-type="aff" rid="Aff40">40</xref></contrib><contrib contrib-type="author"><name><surname>Das</surname><given-names>Sayoni</given-names></name><xref ref-type="aff" rid="Aff41">41</xref></contrib><contrib contrib-type="author"><name><surname>Dawson</surname><given-names>Natalie L.</given-names></name><xref ref-type="aff" rid="Aff41">41</xref></contrib><contrib contrib-type="author"><name><surname>Lee</surname><given-names>David</given-names></name><xref ref-type="aff" rid="Aff41">41</xref></contrib><contrib contrib-type="author"><name><surname>Lees</surname><given-names>Jonathan G.</given-names></name><xref ref-type="aff" rid="Aff41">41</xref></contrib><contrib contrib-type="author"><name><surname>Sillitoe</surname><given-names>Ian</given-names></name><xref ref-type="aff" rid="Aff41">41</xref></contrib><contrib contrib-type="author"><name><surname>Bhat</surname><given-names>Prajwal</given-names></name><xref ref-type="aff" rid="Aff42">42</xref></contrib><contrib contrib-type="author"><name><surname>Nepusz</surname><given-names>Tamás</given-names></name><xref ref-type="aff" rid="Aff43">43</xref></contrib><contrib contrib-type="author"><name><surname>Romero</surname><given-names>Alfonso E.</given-names></name><xref ref-type="aff" rid="Aff44">44</xref></contrib><contrib contrib-type="author"><name><surname>Sasidharan</surname><given-names>Rajkumar</given-names></name><xref ref-type="aff" rid="Aff45">45</xref></contrib><contrib contrib-type="author"><name><surname>Yang</surname><given-names>Haixuan</given-names></name><xref ref-type="aff" rid="Aff46">46</xref></contrib><contrib contrib-type="author"><name><surname>Paccanaro</surname><given-names>Alberto</given-names></name><xref ref-type="aff" rid="Aff44">44</xref></contrib><contrib contrib-type="author"><name><surname>Gillis</surname><given-names>Jesse</given-names></name><xref ref-type="aff" rid="Aff47">47</xref></contrib><contrib contrib-type="author"><name><surname>Sedeño-Cortés</surname><given-names>Adriana E.</given-names></name><xref ref-type="aff" rid="Aff48">48</xref></contrib><contrib contrib-type="author"><name><surname>Pavlidis</surname><given-names>Paul</given-names></name><xref ref-type="aff" rid="Aff49">49</xref></contrib><contrib contrib-type="author"><name><surname>Feng</surname><given-names>Shou</given-names></name><xref ref-type="aff" rid="Aff1">1</xref></contrib><contrib contrib-type="author"><name><surname>Cejuela</surname><given-names>Juan M.</given-names></name><xref ref-type="aff" rid="Aff50">50</xref></contrib><contrib contrib-type="author"><name><surname>Goldberg</surname><given-names>Tatyana</given-names></name><xref ref-type="aff" rid="Aff50">50</xref></contrib><contrib contrib-type="author"><name><surname>Hamp</surname><given-names>Tobias</given-names></name><xref ref-type="aff" rid="Aff50">50</xref></contrib><contrib contrib-type="author"><name><surname>Richter</surname><given-names>Lothar</given-names></name><xref ref-type="aff" rid="Aff50">50</xref></contrib><contrib contrib-type="author"><name><surname>Salamov</surname><given-names>Asaf</given-names></name><xref ref-type="aff" rid="Aff51">51</xref></contrib><contrib contrib-type="author"><name><surname>Gabaldon</surname><given-names>Toni</given-names></name><xref ref-type="aff" rid="Aff52">52</xref><xref ref-type="aff" rid="Aff53">53</xref><xref ref-type="aff" rid="Aff54">54</xref></contrib><contrib contrib-type="author"><name><surname>Marcet-Houben</surname><given-names>Marina</given-names></name><xref ref-type="aff" rid="Aff52">52</xref><xref ref-type="aff" rid="Aff53">53</xref></contrib><contrib contrib-type="author"><name><surname>Supek</surname><given-names>Fran</given-names></name><xref ref-type="aff" rid="Aff53">53</xref><xref ref-type="aff" rid="Aff55">55</xref><xref ref-type="aff" rid="Aff56">56</xref></contrib><contrib contrib-type="author"><name><surname>Gong</surname><given-names>Qingtian</given-names></name><xref ref-type="aff" rid="Aff57">57</xref><xref ref-type="aff" rid="Aff58">58</xref></contrib><contrib contrib-type="author"><name><surname>Ning</surname><given-names>Wei</given-names></name><xref ref-type="aff" rid="Aff57">57</xref><xref ref-type="aff" rid="Aff58">58</xref></contrib><contrib contrib-type="author"><name><surname>Zhou</surname><given-names>Yuanpeng</given-names></name><xref ref-type="aff" rid="Aff57">57</xref><xref ref-type="aff" rid="Aff58">58</xref></contrib><contrib contrib-type="author"><name><surname>Tian</surname><given-names>Weidong</given-names></name><xref ref-type="aff" rid="Aff57">57</xref><xref ref-type="aff" rid="Aff58">58</xref></contrib><contrib contrib-type="author"><name><surname>Falda</surname><given-names>Marco</given-names></name><xref ref-type="aff" rid="Aff59">59</xref></contrib><contrib contrib-type="author"><name><surname>Fontana</surname><given-names>Paolo</given-names></name><xref ref-type="aff" rid="Aff60">60</xref></contrib><contrib contrib-type="author"><name><surname>Lavezzo</surname><given-names>Enrico</given-names></name><xref ref-type="aff" rid="Aff59">59</xref></contrib><contrib contrib-type="author"><name><surname>Toppo</surname><given-names>Stefano</given-names></name><xref ref-type="aff" rid="Aff59">59</xref></contrib><contrib contrib-type="author"><name><surname>Ferrari</surname><given-names>Carlo</given-names></name><xref ref-type="aff" rid="Aff61">61</xref></contrib><contrib contrib-type="author"><name><surname>Giollo</surname><given-names>Manuel</given-names></name><xref ref-type="aff" rid="Aff61">61</xref><xref ref-type="aff" rid="Aff84">84</xref></contrib><contrib contrib-type="author"><name><surname>Piovesan</surname><given-names>Damiano</given-names></name><xref ref-type="aff" rid="Aff61">61</xref></contrib><contrib contrib-type="author"><name><surname>Tosatto</surname><given-names>Silvio C.E.</given-names></name><xref ref-type="aff" rid="Aff61">61</xref></contrib><contrib contrib-type="author"><name><surname>del Pozo</surname><given-names>Angela</given-names></name><xref ref-type="aff" rid="Aff62">62</xref></contrib><contrib contrib-type="author"><name><surname>Fernández</surname><given-names>José M.</given-names></name><xref ref-type="aff" rid="Aff63">63</xref></contrib><contrib contrib-type="author"><name><surname>Maietta</surname><given-names>Paolo</given-names></name><xref ref-type="aff" rid="Aff64">64</xref></contrib><contrib contrib-type="author"><name><surname>Valencia</surname><given-names>Alfonso</given-names></name><xref ref-type="aff" rid="Aff64">64</xref></contrib><contrib contrib-type="author"><name><surname>Tress</surname><given-names>Michael L.</given-names></name><xref ref-type="aff" rid="Aff64">64</xref></contrib><contrib contrib-type="author"><name><surname>Benso</surname><given-names>Alfredo</given-names></name><xref ref-type="aff" rid="Aff65">65</xref></contrib><contrib contrib-type="author"><name><surname>Di Carlo</surname><given-names>Stefano</given-names></name><xref ref-type="aff" rid="Aff65">65</xref></contrib><contrib contrib-type="author"><name><surname>Politano</surname><given-names>Gianfranco</given-names></name><xref ref-type="aff" rid="Aff65">65</xref></contrib><contrib contrib-type="author"><name><surname>Savino</surname><given-names>Alessandro</given-names></name><xref ref-type="aff" rid="Aff65">65</xref></contrib><contrib contrib-type="author"><name><surname>Rehman</surname><given-names>Hafeez Ur</given-names></name><xref ref-type="aff" rid="Aff66">66</xref></contrib><contrib contrib-type="author"><name><surname>Re</surname><given-names>Matteo</given-names></name><xref ref-type="aff" rid="Aff67">67</xref></contrib><contrib contrib-type="author"><name><surname>Mesiti</surname><given-names>Marco</given-names></name><xref ref-type="aff" rid="Aff67">67</xref></contrib><contrib contrib-type="author"><name><surname>Valentini</surname><given-names>Giorgio</given-names></name><xref ref-type="aff" rid="Aff67">67</xref></contrib><contrib contrib-type="author"><name><surname>Bargsten</surname><given-names>Joachim W.</given-names></name><xref ref-type="aff" rid="Aff68">68</xref></contrib><contrib contrib-type="author"><name><surname>van Dijk</surname><given-names>Aalt D. J.</given-names></name><xref ref-type="aff" rid="Aff68">68</xref><xref ref-type="aff" rid="Aff69">69</xref></contrib><contrib contrib-type="author"><name><surname>Gemovic</surname><given-names>Branislava</given-names></name><xref ref-type="aff" rid="Aff70">70</xref></contrib><contrib contrib-type="author"><name><surname>Glisic</surname><given-names>Sanja</given-names></name><xref ref-type="aff" rid="Aff70">70</xref></contrib><contrib contrib-type="author"><name><surname>Perovic</surname><given-names>Vladmir</given-names></name><xref ref-type="aff" rid="Aff70">70</xref></contrib><contrib contrib-type="author"><name><surname>Veljkovic</surname><given-names>Veljko</given-names></name><xref ref-type="aff" rid="Aff70">70</xref></contrib><contrib contrib-type="author"><name><surname>Veljkovic</surname><given-names>Nevena</given-names></name><xref ref-type="aff" rid="Aff70">70</xref></contrib><contrib contrib-type="author"><name><surname>Almeida-e-Silva</surname><given-names>Danillo C.</given-names></name><xref ref-type="aff" rid="Aff71">71</xref></contrib><contrib contrib-type="author"><name><surname>Vencio</surname><given-names>Ricardo Z. N.</given-names></name><xref ref-type="aff" rid="Aff71">71</xref></contrib><contrib contrib-type="author"><name><surname>Sharan</surname><given-names>Malvika</given-names></name><xref ref-type="aff" rid="Aff72">72</xref></contrib><contrib contrib-type="author"><name><surname>Vogel</surname><given-names>Jörg</given-names></name><xref ref-type="aff" rid="Aff72">72</xref></contrib><contrib contrib-type="author"><name><surname>Kansakar</surname><given-names>Lakesh</given-names></name><xref ref-type="aff" rid="Aff73">73</xref></contrib><contrib contrib-type="author"><name><surname>Zhang</surname><given-names>Shanshan</given-names></name><xref ref-type="aff" rid="Aff73">73</xref></contrib><contrib contrib-type="author"><name><surname>Vucetic</surname><given-names>Slobodan</given-names></name><xref ref-type="aff" rid="Aff73">73</xref></contrib><contrib contrib-type="author"><name><surname>Wang</surname><given-names>Zheng</given-names></name><xref ref-type="aff" rid="Aff74">74</xref></contrib><contrib contrib-type="author"><name><surname>Sternberg</surname><given-names>Michael J. E.</given-names></name><xref ref-type="aff" rid="Aff34">34</xref></contrib><contrib contrib-type="author"><name><surname>Wass</surname><given-names>Mark N.</given-names></name><xref ref-type="aff" rid="Aff75">75</xref></contrib><contrib contrib-type="author"><name><surname>Huntley</surname><given-names>Rachael P.</given-names></name><xref ref-type="aff" rid="Aff23">23</xref></contrib><contrib contrib-type="author"><name><surname>Martin</surname><given-names>Maria J.</given-names></name><xref ref-type="aff" rid="Aff23">23</xref></contrib><contrib contrib-type="author"><name><surname>O’Donovan</surname><given-names>Claire</given-names></name><xref ref-type="aff" rid="Aff23">23</xref></contrib><contrib contrib-type="author"><name><surname>Robinson</surname><given-names>Peter N.</given-names></name><xref ref-type="aff" rid="Aff76">76</xref></contrib><contrib contrib-type="author"><name><surname>Moreau</surname><given-names>Yves</given-names></name><xref ref-type="aff" rid="Aff77">77</xref></contrib><contrib contrib-type="author"><name><surname>Tramontano</surname><given-names>Anna</given-names></name><xref ref-type="aff" rid="Aff5">5</xref></contrib><contrib contrib-type="author"><name><surname>Babbitt</surname><given-names>Patricia C.</given-names></name><xref ref-type="aff" rid="Aff78">78</xref></contrib><contrib contrib-type="author"><name><surname>Brenner</surname><given-names>Steven E.</given-names></name><xref ref-type="aff" rid="Aff17">17</xref></contrib><contrib contrib-type="author"><name><surname>Linial</surname><given-names>Michal</given-names></name><xref ref-type="aff" rid="Aff79">79</xref></contrib><contrib contrib-type="author"><name><surname>Orengo</surname><given-names>Christine A.</given-names></name><xref ref-type="aff" rid="Aff41">41</xref></contrib><contrib contrib-type="author"><name><surname>Rost</surname><given-names>Burkhard</given-names></name><xref ref-type="aff" rid="Aff50">50</xref></contrib><contrib contrib-type="author"><name><surname>Greene</surname><given-names>Casey S.</given-names></name><xref ref-type="aff" rid="Aff80">80</xref></contrib><contrib contrib-type="author"><name><surname>Mooney</surname><given-names>Sean D.</given-names></name><xref ref-type="aff" rid="Aff81">81</xref></contrib><contrib contrib-type="author" corresp="yes"><contrib-id contrib-id-type="orcid">http://orcid.org/0000-0002-1789-8000</contrib-id><name><surname>Friedberg</surname><given-names>Iddo</given-names></name><address><email>idoerg@iastate.edu</email></address><xref ref-type="aff" rid="Aff4">4</xref><xref ref-type="aff" rid="Aff82">82</xref></contrib><contrib contrib-type="author" corresp="yes"><name><surname>Radivojac</surname><given-names>Predrag</given-names></name><address><email>predrag@indiana.edu</email></address><xref ref-type="aff" rid="Aff1">1</xref></contrib><aff id="Aff1"><label>1</label>Department of Computer Science and Informatics, Indiana University, Bloomington, IN USA</aff><aff id="Aff2"><label>2</label>Buck Institute for Research on Aging, Novato, CA USA</aff><aff id="Aff3"><label>3</label>Department of Molecular Biophysics and Biochemistry, Yale University, New Haven, CT USA</aff><aff id="Aff4"><label>4</label>Department of Microbiology, Miami University, Oxford, OH USA</aff><aff id="Aff5"><label>5</label>University of Rome, La Sapienza, Rome, Italy</aff><aff id="Aff6"><label>6</label>Computational Bioscience Program, University of Colorado School of Medicine, Aurora, CO USA</aff><aff id="Aff7"><label>7</label>Department of Computer Science, Colorado State University, Fort Collins, CO USA</aff><aff id="Aff8"><label>8</label>Department of Computing and Information Systems, University of Melbourne, Parkville, Victoria, Australia</aff><aff id="Aff9"><label>9</label>Health and Biomedical Informatics Centre, University of Melbourne, Parkville, Victoria, Australia</aff><aff id="Aff10"><label>10</label>Department of Biology, New York University, New York, NY USA</aff><aff id="Aff11"><label>11</label>Social Media and Political Participation Lab, New York University, New York, NY USA</aff><aff id="Aff12"><label>12</label>CY Data Science, New York, NY USA</aff><aff id="Aff13"><label>13</label>Department of Computer Science, New York University, New York, NY USA</aff><aff id="Aff14"><label>14</label>Simons Center for Data Analysis, New York, NY USA</aff><aff id="Aff15"><label>15</label>Center for Genomics and Systems Biology, Department of Biology, New York University, New York, NY USA</aff><aff id="Aff16"><label>16</label>Department of Electrical Engineering and Computer Sciences, University of California Berkeley, Berkeley, CA USA</aff><aff id="Aff17"><label>17</label>Department of Plant and Microbial Biology, University of California Berkeley, Berkeley, CA USA</aff><aff id="Aff18"><label>18</label>Biocomputing Group, BiGeA, University of Bologna, Bologna, Italy</aff><aff id="Aff19"><label>19</label>Computer Science Department, University of Missouri, Columbia, MO USA</aff><aff id="Aff20"><label>20</label>ETH Zurich, Zurich, Switzerland</aff><aff id="Aff21"><label>21</label>Swiss Institute of Bioinformatics, Zurich, Switzerland</aff><aff id="Aff22"><label>22</label>Bioinformatics Group, Department of Computer Science, University College London, London, UK</aff><aff id="Aff23"><label>23</label>European Molecular Biology Laboratory, European Bioinformatics Institute, Cambridge, UK</aff><aff id="Aff24"><label>24</label>Department of Information Technology, University of Turku, Turku, Finland</aff><aff id="Aff25"><label>25</label>University of Turku Graduate School, University of Turku, Turku, Finland</aff><aff id="Aff26"><label>26</label>Turku Centre for Computer Science, Turku, Finland</aff><aff id="Aff27"><label>27</label>University of Bristol, Bristol, UK</aff><aff id="Aff28"><label>28</label>Institute of Biotechnology, University of Helsinki, Helsinki, Finland</aff><aff id="Aff29"><label>29</label>Institute of Information Science, Academia Sinica, Taipei, Taiwan</aff><aff id="Aff30"><label>30</label>Department of Computational Science and Engineering, North Carolina A&T State University, Greensboro, NC USA</aff><aff id="Aff31"><label>31</label>Department of Computer Science, Purdue University, West Lafayette, IN USA</aff><aff id="Aff32"><label>32</label>Department of Biological Chemistry, Institute of Life Sciences, The Hebrew University of Jerusalem, Jerusalem, Israel</aff><aff id="Aff33"><label>33</label>School of Computer Science and Engineering, The Hebrew University of Jerusalem, Jerusalem, Israel</aff><aff id="Aff34"><label>34</label>Centre for Integrative Systems Biology and Bioinformatics, Department of Life Sciences, Imperial College London, London, UK</aff><aff id="Aff35"><label>35</label>Centre for Cardiovascular Genetics, Institute of Cardiovascular Science, University College London, London, UK</aff><aff id="Aff36"><label>36</label>Department of Electrical Engineering, STADIUS Center for Dynamical Systems, Signal Processing and Data Analytics, KU Leuven, Leuven, Belgium</aff><aff id="Aff37"><label>37</label>iMinds Department Medical Information Technologies, Leuven, Belgium</aff><aff id="Aff38"><label>38</label>Inserm UMR-S1052, CNRS UMR5286, Cancer Research Centre of Lyon, Lyon, France</aff><aff id="Aff39"><label>39</label>Université de Lyon 1, Villeurbanne, France</aff><aff id="Aff40"><label>40</label>Centre Léon Bérard, Lyon, France</aff><aff id="Aff41"><label>41</label>Institute of Structural and Molecular Biology, University College London, London, UK</aff><aff id="Aff42"><label>42</label>Cerenode Inc., Boston, MA USA</aff><aff id="Aff43"><label>43</label>Molde University College, Molde, Norway</aff><aff id="Aff44"><label>44</label>Department of Computer Science, Centre for Systems and Synthetic Biology, Royal Holloway University of London, Egham, UK</aff><aff id="Aff45"><label>45</label>Department of Molecular, Cell and Developmental Biology, University of California at Los Angeles, Los Angeles, CA USA</aff><aff id="Aff46"><label>46</label>School of Mathematics, Statistics and Applied Mathematics, National University of Ireland, Galway, Ireland</aff><aff id="Aff47"><label>47</label>Stanley Institute for Cognitive Genomics Cold Spring Harbor Laboratory, New York, NY USA</aff><aff id="Aff48"><label>48</label>Graduate Program in Bioinformatics, University of British Columbia, Vancouver, Canada</aff><aff id="Aff49"><label>49</label>Department of Psychiatry and Michael Smith Laboratories, University of British Columbia, Vancouver, Canada</aff><aff id="Aff50"><label>50</label>Department for Bioinformatics and Computational Biology-I12, Technische Universität München, Garching, Germany</aff><aff id="Aff51"><label>51</label>DOE Joint Genome Institute, Walnut Creek, CA USA</aff><aff id="Aff52"><label>52</label>Bioinformatics and Genomics, Centre for Genomic Regulation, Barcelona, Spain</aff><aff id="Aff53"><label>53</label>Universitat Pompeu Fabra, Barcelona, Spain</aff><aff id="Aff54"><label>54</label>Institució Catalana de Recerca i Estudis Avançats, Barcelona, Spain</aff><aff id="Aff55"><label>55</label>Division of Electronics, Rudjer Boskovic Institute, Zagreb, Croatia</aff><aff id="Aff56"><label>56</label>EMBL/CRG Systems Biology Research Unit, Centre for Genomic Regulation, Barcelona, Spain</aff><aff id="Aff57"><label>57</label>State Key Laboratory of Genetic Engineering, Collaborative Innovation Center of Genetics and Development, Department of Biostatistics and Computational Biology, School of Life Science, Fudan University, Shanghai, China</aff><aff id="Aff58"><label>58</label>Children’s Hospital of Fudan University, Shanghai, China</aff><aff id="Aff59"><label>59</label>Department of Molecular Medicine, University of Padua, Padua, Italy</aff><aff id="Aff60"><label>60</label>Research and Innovation Center, Edmund Mach Foundation, San Michele all’Adige, Italy</aff><aff id="Aff61"><label>61</label>Department of Information Engineering, University of Padua, Padova, Italy</aff><aff id="Aff62"><label>62</label>Instituto De Genetica Medica y Molecular, Hospital Universitario de La Paz, Madrid, Spain</aff><aff id="Aff63"><label>63</label>Spanish National Bioinformatics Institute, Spanish National Cancer Research Institute, Madrid, Spain</aff><aff id="Aff64"><label>64</label>Structural and Computational Biology Programme, Spanish National Cancer Research Institute, Madrid, Spain</aff><aff id="Aff65"><label>65</label>Control and Computer Engineering Department, Politecnico di Torino, Torino, Italy</aff><aff id="Aff66"><label>66</label>National University of Computer & Emerging Sciences, Islamabad, Pakistan</aff><aff id="Aff67"><label>67</label>Anacleto Lab, Dipartimento di informatica, Università degli Studi di Milano, Milan, Italy</aff><aff id="Aff68"><label>68</label>Applied Bioinformatics, Bioscience, Wageningen University and Research Centre, Wageningen, Netherlands</aff><aff id="Aff69"><label>69</label>Biometris, Wageningen University, Wageningen, Netherlands</aff><aff id="Aff70"><label>70</label>Center for Multidisciplinary Research, Institute of Nuclear Sciences Vinca, University of Belgrade, Belgrade, Serbia</aff><aff id="Aff71"><label>71</label>Department of Computing and Mathematics FFCLRP-USP, University of Sao Paulo, Ribeirao Preto, Brazil</aff><aff id="Aff72"><label>72</label>Institute for Molecular Infection Biology, University of Würzburg, Würzburg, Germany</aff><aff id="Aff73"><label>73</label>Department of Computer and Information Sciences, Temple University, Philadelphia, PA USA</aff><aff id="Aff74"><label>74</label>University of Southern Mississippi, Hattiesburg, MS USA</aff><aff id="Aff75"><label>75</label>School of Biosciences, University of Kent, Canterbury, Kent, UK</aff><aff id="Aff76"><label>76</label>Institut für Medizinische Genetik und Humangenetik, Charité - Universitätsmedizin Berlin, Berlin, Germany</aff><aff id="Aff77"><label>77</label>Department of Electrical Engineering ESAT-SCD and IBBT-KU Leuven Future Health Department, Katholieke Universiteit Leuven, Leuven, Belgium</aff><aff id="Aff78"><label>78</label>California Institute for Quantitative Biosciences, University of California San Francisco, San Francisco, CA USA</aff><aff id="Aff79"><label>79</label>Department of Chemical Biology, The Hebrew University of Jerusalem, Jerusalem, Israel</aff><aff id="Aff80"><label>80</label>Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania, Philadelphia, PA USA</aff><aff id="Aff81"><label>81</label>Department of Biomedical Informatics and Medical Education, University of Washington, Seattle, WA USA</aff><aff id="Aff82"><label>82</label>Department of Computer Science, Miami University, Oxford, OH USA</aff><aff id="Aff83"><label>83</label>Department of Veterinary Microbiology and Preventive Medicine, Iowa State University, Ames, IA USA</aff><aff id="Aff84"><label>84</label>Department of Biomedical Sciences, University of Padua, Padova, Italy</aff><aff id="Aff85"><label>85</label>Department of Biological Sciences, Purdue University, West Lafayette, IN USA</aff><aff id="Aff86"><label>86</label>Department of Biological and Environmental Sciences, Universitity of Helsinki, Helsinki, Finland</aff><aff id="Aff87"><label>87</label>University of Lausanne, Lausanne, Switzerland</aff><aff id="Aff88"><label>88</label>Swiss Institute of Bioinformatics, Lausanne, Switzerland</aff></contrib-group><pub-date pub-type="epub"><day>7</day><month>9</month><year>2016</year></pub-date><pub-date pub-type="pmc-release"><day>7</day><month>9</month><year>2016</year></pub-date><pub-date pub-type="collection"><year>2016</year></pub-date><volume>17</volume><issue>1</issue><elocation-id>184</elocation-id><history><date date-type="received"><day>26</day><month>10</month><year>2015</year></date><date date-type="accepted"><day>4</day><month>8</month><year>2016</year></date></history><permissions><copyright-statement>© The Author(s) 2016</copyright-statement><license license-type="OpenAccess"><license-p><bold>Open Access</bold> This article is distributed under the terms of the Creative Commons Attribution 4.0 International License (<ext-link ext-link-type="uri" xlink:href="http://creativecommons.org/licenses/by/4.0/">http://creativecommons.org/licenses/by/4.0/</ext-link>), which permits unrestricted use, distribution, and reproduction in any medium, provided you give appropriate credit to the original author(s) and the source, provide a link to the Creative Commons license, and indicate if changes were made. The Creative Commons Public Domain Dedication waiver (<ext-link ext-link-type="uri" xlink:href="http://creativecommons.org/publicdomain/zero/1.0/">http://creativecommons.org/publicdomain/zero/1.0/</ext-link>) applies to the data made available in this article, unless otherwise stated.</license-p></license></permissions><abstract id="Abs1"><sec><title>Background</title><p>A major bottleneck in our understanding of the molecular underpinnings of life is the assignment of function to proteins. While molecular experiments provide the most reliable annotation of proteins, their relatively low throughput and restricted purview have led to an increasing role for computational function prediction. However, assessing methods for protein function prediction and tracking progress in the field remain challenging.</p></sec><sec><title>Results</title><p>We conducted the second critical assessment of functional annotation (CAFA), a timed challenge to assess computational methods that automatically assign protein function. We evaluated 126 methods from 56 research groups for their ability to predict biological functions using Gene Ontology and gene-disease associations using Human Phenotype Ontology on a set of 3681 proteins from 18 species. CAFA2 featured expanded analysis compared with CAFA1, with regards to data set size, variety, and assessment metrics. To review progress in the field, the analysis compared the best methods from CAFA1 to those of CAFA2.</p></sec><sec><title>Conclusions</title><p>The top-performing methods in CAFA2 outperformed those from CAFA1. This increased accuracy can be attributed to a combination of the growing number of experimental annotations and improved methods for function prediction. The assessment also revealed that the definition of top-performing algorithms is ontology specific, that different performance metrics can be used to probe the nature of accurate predictions, and the relative diversity of predictions in the biological process and human phenotype ontologies. While there was methodological improvement between CAFA1 and CAFA2, the interpretation of results and usefulness of individual methods remain context-dependent.</p></sec><sec><title>Electronic supplementary material</title><p>The online version of this article (doi:10.1186/s13059-016-1037-6) contains supplementary material, which is available to authorized users.</p></sec></abstract><kwd-group xml:lang="en"><title>Keywords</title><kwd>Protein function prediction</kwd><kwd>Disease gene prioritization</kwd></kwd-group><funding-group><award-group><funding-source><institution-wrap><institution-id institution-id-type="FundRef">http://dx.doi.org/10.13039/100000076</institution-id><institution>Directorate for Biological Sciences</institution></institution-wrap></funding-source><award-id>1458359</award-id><principal-award-recipient><name><surname>Friedberg</surname><given-names>Iddo</given-names></name></principal-award-recipient></award-group><award-group><funding-source><institution-wrap><institution-id institution-id-type="FundRef">http://dx.doi.org/10.13039/100000076</institution-id><institution>Directorate for Biological Sciences</institution></institution-wrap></funding-source><award-id>1458477</award-id><principal-award-recipient><name><surname>Radivojac</surname><given-names>Predrag</given-names></name></principal-award-recipient></award-group></funding-group><custom-meta-group><custom-meta><meta-name>issue-copyright-statement</meta-name><meta-value>© The Author(s) 2016</meta-value></custom-meta></custom-meta-group></article-meta></front><body><sec id="Sec1"><title>Background</title><p>Accurate computer-generated functional annotations of biological macromolecules allow biologists to rapidly generate testable hypotheses about the roles that newly identified proteins play in processes or pathways. They also allow them to reason about new species based on the observed functional repertoire associated with their genes. However, protein function prediction is an open research problem and it is not yet clear which tools are best for predicting function. At the same time, critically evaluating these tools and understanding the landscape of the function prediction field is a challenging task that extends beyond the capabilities of a single lab.</p><p>Assessments and challenges have a successful history of driving the development of new methods in the life sciences by independently assessing performance and providing discussion forums for the researchers [<xref ref-type="bibr" rid="CR1">1</xref>]. In 2010–2011, we organized the first critical assessment of functional annotation (CAFA) challenge to evaluate methods for the automated annotation of protein function and to assess the progress in method development in the first decade of the 2000s [<xref ref-type="bibr" rid="CR2">2</xref>]. The challenge used a time-delayed evaluation of predictions for a large set of target proteins without any experimental functional annotation. A subset of these target proteins accumulated experimental annotations after the predictions were submitted and was used to estimate the performance accuracy. The estimated performance was subsequently used to draw conclusions about the status of the field.</p><p>The first CAFA (CAFA1) showed that advanced methods for the prediction of Gene Ontology (GO) terms [<xref ref-type="bibr" rid="CR3">3</xref>] significantly outperformed a straightforward application of function transfer by local sequence similarity. In addition to validating investment in the development of new methods, CAFA1 also showed that using machine learning to integrate multiple sequence hits and multiple data types tends to perform well. However, CAFA1 also identified challenges for experimentalists, biocurators, and computational biologists. These challenges include the choice of experimental techniques and proteins in functional studies and curation, the structure and status of biomedical ontologies, the lack of comprehensive systems data that are necessary for accurate prediction of complex biological concepts, as well as limitations of evaluation metrics [<xref ref-type="bibr" rid="CR2">2</xref>, <xref ref-type="bibr" rid="CR4">4</xref>–<xref ref-type="bibr" rid="CR7">7</xref>]. Overall, by establishing the state-of-the-art in the field and identifying challenges, CAFA1 set the stage for quantifying progress in the field of protein function prediction over time.</p><p>In this study, we report on the major outcomes of the second CAFA experiment, CAFA2, that was organized and conducted in 2013–2014, exactly 3 years after the original experiment. We were motivated to evaluate the progress in method development for function prediction as well as to expand the experiment to new ontologies. The CAFA2 experiment also greatly expanded the performance analysis to new types of evaluation and included new performance metrics. By surveying the state of the field, we aim to help all direct and indirect users of computational function prediction software develop intuition for the quality, robustness, and reliability of these predictions.</p></sec><sec id="Sec2"><title>Methods</title><sec id="Sec3"><title>Experiment overview</title><p>The time line for the second CAFA experiment followed that of the first experiment and is illustrated in Fig. <xref rid="Fig1" ref-type="fig">1</xref>. Briefly, CAFA2 was announced in July 2013 and officially started in September 2013, when 100,816 <italic>target sequences</italic> from 27 species were made available to the community. Teams were required to submit prediction scores within the (0,1] range for each protein–term pair they chose to predict on. The submission deadline for depositing these predictions was set for January 2014 (time point <italic>t</italic><sub>0</sub>). We then waited until September 2014 (time point <italic>t</italic><sub>1</sub>) for new experimental annotations to accumulate on the target proteins and assessed the performance of the prediction methods. We will refer to the set of all experimentally annotated proteins available at <italic>t</italic><sub>0</sub> as the <italic>training set</italic> and to a subset of target proteins that accumulated experimental annotations during (<italic>t</italic><sub>0</sub>,<italic>t</italic><sub>1</sub>] and used for evaluation as the <italic>benchmark set</italic>. It is important to note that the benchmark proteins and the resulting analysis vary based on the selection of time point <italic>t</italic><sub>1</sub>. For example, a preliminary analysis of the CAFA2 experiment was provided during the Automated Function Prediction Special Interest Group (AFP-SIG) meeting at the Intelligent Systems for Molecular Biology (ISMB) conference in July 2014.
<fig id="Fig1"><label>Fig. 1</label><caption><p>Time line for the CAFA2 experiment</p></caption><graphic xlink:href="13059_2016_1037_Fig1_HTML" id="MO1"></graphic></fig></p><p>The participating methods were evaluated according to their ability to predict terms in GO [<xref ref-type="bibr" rid="CR3">3</xref>] and Human Phenotype Ontology (HPO) [<xref ref-type="bibr" rid="CR8">8</xref>]. In contrast with CAFA1, where the evaluation was carried out only for the Molecular Function Ontology (MFO) and Biological Process Ontology (BPO), in CAFA2 we also assessed the performance for the prediction of Cellular Component Ontology (CCO) terms in GO. The set of human proteins was further used to evaluate methods according to their ability to associate these proteins with disease terms from HPO, which included all sub-classes of the term HP:0000118, “Phenotypic abnormality”.</p><p>In total, 56 groups submitting 126 methods participated in CAFA2. From those, 125 methods made valid predictions on a sufficient number of sequences. Further, 121 methods submitted predictions for at least one of the GO benchmarks, while 30 methods participated in the disease gene prediction tasks using HPO.</p></sec><sec id="Sec4"><title>Evaluation</title><p>The CAFA2 experiment expanded the assessment of computational function prediction compared with CAFA1. This includes the increased number of targets, benchmarks, ontologies, and method comparison metrics.</p><p>We distinguish between two major types of method evaluation. The first, <italic>protein-centric evaluation</italic>, assesses performance accuracy of methods that predict all ontological terms associated with a given protein sequence. The second type, <italic>term-centric evaluation</italic>, assesses performance accuracy of methods that predict if a single ontology term of interest is associated with a given protein sequence [<xref ref-type="bibr" rid="CR2">2</xref>]. The protein-centric evaluation can be viewed as a multi-label or structured-output learning problem of predicting a set of terms or a directed acyclic graph (a subgraph of the ontology) for a given protein. Because the ontologies contain many terms, the output space in this setting is extremely large and the evaluation metrics must incorporate similarity functions between groups of mutually interdependent terms (directed acyclic graphs). In contrast, the term-centric evaluation is an example of binary classification, where a given ontology term is assigned (or not) to an input protein sequence. These methods are particularly common in disease gene prioritization [<xref ref-type="bibr" rid="CR9">9</xref>]. Put otherwise, a protein-centric evaluation considers a ranking of ontology terms for a given protein, whereas the term-centric evaluation considers a ranking of protein sequences for a given ontology term.</p><p>Both types of evaluation have merits in assessing performance. This is partly due to the statistical dependency between ontology terms, the statistical dependency among protein sequences, and also the incomplete and biased nature of the experimental annotation of protein function [<xref ref-type="bibr" rid="CR6">6</xref>]. In CAFA2, we provide both types of evaluation, but we emphasize the protein-centric scenario for easier comparisons with CAFA1. We also draw important conclusions regarding method assessment in these two scenarios.</p><sec id="Sec5"><title>No-knowledge and limited-knowledge benchmark sets</title><p>In CAFA1, a protein was eligible to be in the benchmark set if it had not had any experimentally verified annotations in any of the GO ontologies at time <italic>t</italic><sub>0</sub> but accumulated at least one functional term with an experimental evidence code between <italic>t</italic><sub>0</sub> and <italic>t</italic><sub>1</sub>; we refer to such benchmark proteins as <italic>no-knowledge</italic> benchmarks. In CAFA2 we introduced proteins with <italic>limited knowledge</italic>, which are those that had been experimentally annotated in one or two GO ontologies (but not in all three) at time <italic>t</italic><sub>0</sub>. For example, for the performance evaluation in MFO, a protein without any annotation in MFO prior to the submission deadline was allowed to have experimental annotations in BPO and CCO.</p><p>During the growth phase, the no-knowledge targets that have acquired experimental annotations in one or more ontologies became benchmarks in those ontologies. The limited-knowledge targets that have acquired additional annotations became benchmarks only for those ontologies for which there were no prior experimental annotations. The reason for using limited-knowledge targets was to identify whether the correlations between experimental annotations across ontologies can be exploited to improve function prediction.</p><p>The selection of benchmark proteins for evaluating HPO-term predictors was separated from the GO analyses. We created only a no-knowledge benchmark set in the HPO category.</p></sec><sec id="Sec6"><title>Partial and full evaluation modes</title><p>Many function prediction methods apply only to certain types of proteins, such as proteins for which 3D structure data are available, proteins from certain taxa, or specific subcellular localizations. To accommodate these methods, CAFA2 provided predictors with an option of choosing a subset of the targets to predict on as long as they computationally annotated at least 5,000 targets, of which at least ten accumulated experimental terms. We refer to the assessment mode in which the predictions were evaluated only on those benchmarks for which a model made at least one prediction at any threshold as <italic>partial evaluation mode</italic>. In contrast, the <italic>full evaluation mode</italic> corresponds to the same type of assessment performed in CAFA1 where all benchmark proteins were used for the evaluation and methods were penalized for not making predictions.</p><p>In most cases, for each benchmark category, we have two types of benchmarks, no-knowledge and limited-knowledge, and two modes of evaluation, full mode and partial mode. Exceptions are all HPO categories that only have no-knowledge benchmarks. The full mode is appropriate for comparisons of general-purpose methods designed to make predictions on any protein, while the partial mode gives an idea of how well each method performs on a self-selected subset of targets.</p></sec><sec id="Sec7"><title>Evaluation metrics</title><p>Precision–recall curves and remaining uncertainty–misinformation curves were used as the two chief metrics in the protein-centric mode [<xref ref-type="bibr" rid="CR10">10</xref>]. We also provide a single measure for evaluation of both types of curves as a real-valued scalar to compare methods; however, we note that any choice of a single point on those curves may not match the intended application objectives for a given algorithm. Thus, a careful understanding of the evaluation metrics used in CAFA is necessary to properly interpret the results.</p><p>Precision (pr), recall (rc), and the resulting <italic>F</italic><sub>max</sub> are defined as
<disp-formula id="Equa"><alternatives><tex-math id="M1">\documentclass[12pt]{minimal}
\usepackage{amsmath}
\usepackage{wasysym}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{amsbsy}
\usepackage{mathrsfs}
\usepackage{upgreek}
\setlength{\oddsidemargin}{-69pt}
\begin{document} $$\begin{array}{@{}rcl@{}} \text{pr}(\tau) &=& \frac{1}{m(\tau)}\sum\limits_{i=1}^{m(\tau)} \frac{{\sum\nolimits}_{f} \mathbbm{1}\left(f \in P_{i}(\tau) \wedge f \in T_{i}\right)}{\sum_{f} \mathbbm{1}\left(f \in P_{i}(\tau) \right)},\\ \text{rc}(\tau) &=& \frac{1}{n_{e}}\sum\limits_{i=1}^{n_{e}} \frac{{\sum\nolimits}_{f} \mathbbm{1}\left(f \in {P}_{i}(\tau) \wedge f \in T_{i}\right)}{{\sum\nolimits}_{f} \mathbbm{1}\left(f \in T_{i} \right)}, \\ F_{\max} &=& \max_{\tau} \left\{ \frac{2\cdot \text{pr}(\tau)\cdot \text{rc}(\tau)}{\text{pr}(\tau) + \text{rc}(\tau)} \right\}, \end{array} $$ \end{document}</tex-math><mml:math id="M2"><mml:mtable class="eqnarray" columnalign="left center right"><mml:mtr><mml:mtd class="eqnarray-1"><mml:mtext>pr</mml:mtext><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo></mml:mtd><mml:mtd class="eqnarray-2"><mml:mo>=</mml:mo></mml:mtd><mml:mtd class="eqnarray-3"><mml:mfrac><mml:mrow><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:mi>m</mml:mi><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo></mml:mrow></mml:mfrac><mml:munderover accent="false" accentunder="false"><mml:mrow><mml:mo>∑</mml:mo></mml:mrow><mml:mrow><mml:mi>i</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:mi>m</mml:mi><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo></mml:mrow></mml:munderover><mml:mfrac><mml:mrow><mml:msub><mml:mrow><mml:mo>∑</mml:mo></mml:mrow><mml:mrow><mml:mi>f</mml:mi></mml:mrow></mml:msub><mml:mi>1</mml:mi><mml:mfenced close=")" open="(" separators=""><mml:mrow><mml:mi>f</mml:mi><mml:mo>∈</mml:mo><mml:msub><mml:mrow><mml:mi>P</mml:mi></mml:mrow><mml:mrow><mml:mi>i</mml:mi></mml:mrow></mml:msub><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo><mml:mo>∧</mml:mo><mml:mi>f</mml:mi><mml:mo>∈</mml:mo><mml:msub><mml:mrow><mml:mi>T</mml:mi></mml:mrow><mml:mrow><mml:mi>i</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mfenced></mml:mrow><mml:mrow><mml:munder><mml:mrow><mml:mo>∑</mml:mo></mml:mrow><mml:mrow><mml:mi>f</mml:mi></mml:mrow></mml:munder><mml:mi>1</mml:mi><mml:mfenced close=")" open="(" separators=""><mml:mrow><mml:mi>f</mml:mi><mml:mo>∈</mml:mo><mml:msub><mml:mrow><mml:mi>P</mml:mi></mml:mrow><mml:mrow><mml:mi>i</mml:mi></mml:mrow></mml:msub><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo></mml:mrow></mml:mfenced></mml:mrow></mml:mfrac><mml:mo>,</mml:mo></mml:mtd><mml:mtd><mml:mtext></mml:mtext></mml:mtd></mml:mtr><mml:mtr><mml:mtd class="eqnarray-1"><mml:mtext>rc</mml:mtext><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo></mml:mtd><mml:mtd class="eqnarray-2"><mml:mo>=</mml:mo></mml:mtd><mml:mtd class="eqnarray-3"><mml:mfrac><mml:mrow><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi>n</mml:mi></mml:mrow><mml:mrow><mml:mi>e</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mfrac><mml:munderover accent="false" accentunder="false"><mml:mrow><mml:mo>∑</mml:mo></mml:mrow><mml:mrow><mml:mi>i</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi>n</mml:mi></mml:mrow><mml:mrow><mml:mi>e</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:munderover><mml:mfrac><mml:mrow><mml:msub><mml:mrow><mml:mo>∑</mml:mo></mml:mrow><mml:mrow><mml:mi>f</mml:mi></mml:mrow></mml:msub><mml:mi>1</mml:mi><mml:mfenced close=")" open="(" separators=""><mml:mrow><mml:mi>f</mml:mi><mml:mo>∈</mml:mo><mml:msub><mml:mrow><mml:mi>P</mml:mi></mml:mrow><mml:mrow><mml:mi>i</mml:mi></mml:mrow></mml:msub><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo><mml:mo>∧</mml:mo><mml:mi>f</mml:mi><mml:mo>∈</mml:mo><mml:msub><mml:mrow><mml:mi>T</mml:mi></mml:mrow><mml:mrow><mml:mi>i</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mfenced></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mo>∑</mml:mo></mml:mrow><mml:mrow><mml:mi>f</mml:mi></mml:mrow></mml:msub><mml:mi>1</mml:mi><mml:mfenced close=")" open="(" separators=""><mml:mrow><mml:mi>f</mml:mi><mml:mo>∈</mml:mo><mml:msub><mml:mrow><mml:mi>T</mml:mi></mml:mrow><mml:mrow><mml:mi>i</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mfenced></mml:mrow></mml:mfrac><mml:mo>,</mml:mo></mml:mtd><mml:mtd><mml:mtext></mml:mtext></mml:mtd></mml:mtr><mml:mtr><mml:mtd class="eqnarray-1"><mml:msub><mml:mrow><mml:mi>F</mml:mi></mml:mrow><mml:mrow><mml:mo>max</mml:mo></mml:mrow></mml:msub></mml:mtd><mml:mtd class="eqnarray-2"><mml:mo>=</mml:mo></mml:mtd><mml:mtd class="eqnarray-3"><mml:munder><mml:mrow><mml:mo>max</mml:mo></mml:mrow><mml:mrow><mml:mi>τ</mml:mi></mml:mrow></mml:munder><mml:mfenced close="}" open="{" separators=""><mml:mrow><mml:mfrac><mml:mrow><mml:mn>2</mml:mn><mml:mo>·</mml:mo><mml:mtext>pr</mml:mtext><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo><mml:mo>·</mml:mo><mml:mtext>rc</mml:mtext><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo></mml:mrow><mml:mrow><mml:mtext>pr</mml:mtext><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo><mml:mo>+</mml:mo><mml:mtext>rc</mml:mtext><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo></mml:mrow></mml:mfrac></mml:mrow></mml:mfenced><mml:mo>,</mml:mo></mml:mtd><mml:mtd><mml:mtext></mml:mtext></mml:mtd></mml:mtr></mml:mtable></mml:math><graphic xlink:href="13059_2016_1037_Article_Equa.gif" position="anchor"></graphic></alternatives></disp-formula></p><p>where <italic>P</italic><sub><italic>i</italic></sub>(<italic>τ</italic>) denotes the set of terms that have predicted scores greater than or equal to <italic>τ</italic> for a protein sequence <italic>i</italic>, <italic>T</italic><sub><italic>i</italic></sub> denotes the corresponding ground-truth set of terms for that sequence, <italic>m</italic>(<italic>τ</italic>) is the number of sequences with at least one predicted score greater than or equal to <italic>τ</italic>, <inline-formula id="IEq1"><alternatives><tex-math id="M3">\documentclass[12pt]{minimal}
\usepackage{amsmath}
\usepackage{wasysym}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{amsbsy}
\usepackage{mathrsfs}
\usepackage{upgreek}
\setlength{\oddsidemargin}{-69pt}
\begin{document}$\mathbbm {1}\left (\cdot \right)$\end{document}</tex-math><mml:math id="M4"><mml:mi>1</mml:mi><mml:mfenced close=")" open="(" separators=""><mml:mrow><mml:mo>·</mml:mo></mml:mrow></mml:mfenced></mml:math><inline-graphic xlink:href="13059_2016_1037_Article_IEq1.gif"></inline-graphic></alternatives></inline-formula> is an indicator function, and <italic>n</italic><sub><italic>e</italic></sub> is the number of targets used in a particular mode of evaluation. In the full evaluation mode <italic>n</italic><sub><italic>e</italic></sub>=<italic>n</italic>, the number of benchmark proteins, whereas in the partial evaluation mode <italic>n</italic><sub><italic>e</italic></sub>=<italic>m</italic>(0), i.e., the number of proteins that were chosen to be predicted using the particular method. For each method, we refer to <italic>m</italic>(0)/<italic>n</italic> as the <italic>coverage</italic> because it provides the fraction of benchmark proteins on which the method made any predictions.</p><p>The remaining uncertainty (ru), misinformation (mi), and the resulting minimum semantic distance (<italic>S</italic><sub>min</sub>) are defined as
<disp-formula id="Equb"><alternatives><tex-math id="M5">\documentclass[12pt]{minimal}
\usepackage{amsmath}
\usepackage{wasysym}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{amsbsy}
\usepackage{mathrsfs}
\usepackage{upgreek}
\setlength{\oddsidemargin}{-69pt}
\begin{document} $$\begin{array}{@{}rcl@{}} \text{ru}(\tau) &=& \frac{1}{n_{e}}\sum\limits_{i=1}^{n_{e}} \sum\limits_{f} \text{ic}(f) \cdot \mathbbm{1}\left(f \notin P_{i}(\tau) \wedge f \in T_{i} \right),\\ \text{mi}(\tau) &=& \frac{1}{n_{e}}\sum\limits_{i=1}^{n_{e}} \sum\limits_{f} \text{ic}(f) \cdot \mathbbm{1}\left(f \in P_{i}(\tau) \wedge f \notin T_{i} \right), \\ S_{\min} &=& \min_{\tau}\left\{ \sqrt{\text{ru}(\tau)^{2} + \text{mi}(\tau)^{2}} \right\}, \end{array} $$ \end{document}</tex-math><mml:math id="M6"><mml:mtable class="eqnarray" columnalign="left center right"><mml:mtr><mml:mtd class="eqnarray-1"><mml:mtext>ru</mml:mtext><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo></mml:mtd><mml:mtd class="eqnarray-2"><mml:mo>=</mml:mo></mml:mtd><mml:mtd class="eqnarray-3"><mml:mfrac><mml:mrow><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi>n</mml:mi></mml:mrow><mml:mrow><mml:mi>e</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mfrac><mml:munderover accent="false" accentunder="false"><mml:mrow><mml:mo>∑</mml:mo></mml:mrow><mml:mrow><mml:mi>i</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi>n</mml:mi></mml:mrow><mml:mrow><mml:mi>e</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:munderover><mml:munder><mml:mrow><mml:mo>∑</mml:mo></mml:mrow><mml:mrow><mml:mi>f</mml:mi></mml:mrow></mml:munder><mml:mtext>ic</mml:mtext><mml:mo>(</mml:mo><mml:mi>f</mml:mi><mml:mo>)</mml:mo><mml:mo>·</mml:mo><mml:mi>1</mml:mi><mml:mfenced close=")" open="(" separators=""><mml:mrow><mml:mi>f</mml:mi><mml:mo>∉</mml:mo><mml:msub><mml:mrow><mml:mi>P</mml:mi></mml:mrow><mml:mrow><mml:mi>i</mml:mi></mml:mrow></mml:msub><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo><mml:mo>∧</mml:mo><mml:mi>f</mml:mi><mml:mo>∈</mml:mo><mml:msub><mml:mrow><mml:mi>T</mml:mi></mml:mrow><mml:mrow><mml:mi>i</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mfenced><mml:mo>,</mml:mo></mml:mtd><mml:mtd><mml:mtext></mml:mtext></mml:mtd></mml:mtr><mml:mtr><mml:mtd class="eqnarray-1"><mml:mtext>mi</mml:mtext><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo></mml:mtd><mml:mtd class="eqnarray-2"><mml:mo>=</mml:mo></mml:mtd><mml:mtd class="eqnarray-3"><mml:mfrac><mml:mrow><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi>n</mml:mi></mml:mrow><mml:mrow><mml:mi>e</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mfrac><mml:munderover accent="false" accentunder="false"><mml:mrow><mml:mo>∑</mml:mo></mml:mrow><mml:mrow><mml:mi>i</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:msub><mml:mrow><mml:mi>n</mml:mi></mml:mrow><mml:mrow><mml:mi>e</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:munderover><mml:munder><mml:mrow><mml:mo>∑</mml:mo></mml:mrow><mml:mrow><mml:mi>f</mml:mi></mml:mrow></mml:munder><mml:mtext>ic</mml:mtext><mml:mo>(</mml:mo><mml:mi>f</mml:mi><mml:mo>)</mml:mo><mml:mo>·</mml:mo><mml:mi>1</mml:mi><mml:mfenced close=")" open="(" separators=""><mml:mrow><mml:mi>f</mml:mi><mml:mo>∈</mml:mo><mml:msub><mml:mrow><mml:mi>P</mml:mi></mml:mrow><mml:mrow><mml:mi>i</mml:mi></mml:mrow></mml:msub><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo><mml:mo>∧</mml:mo><mml:mi>f</mml:mi><mml:mo>∉</mml:mo><mml:msub><mml:mrow><mml:mi>T</mml:mi></mml:mrow><mml:mrow><mml:mi>i</mml:mi></mml:mrow></mml:msub></mml:mrow></mml:mfenced><mml:mo>,</mml:mo></mml:mtd><mml:mtd><mml:mtext></mml:mtext></mml:mtd></mml:mtr><mml:mtr><mml:mtd class="eqnarray-1"><mml:msub><mml:mrow><mml:mi>S</mml:mi></mml:mrow><mml:mrow><mml:mo>min</mml:mo></mml:mrow></mml:msub></mml:mtd><mml:mtd class="eqnarray-2"><mml:mo>=</mml:mo></mml:mtd><mml:mtd class="eqnarray-3"><mml:munder><mml:mrow><mml:mo>min</mml:mo></mml:mrow><mml:mrow><mml:mi>τ</mml:mi></mml:mrow></mml:munder><mml:mfenced close="}" open="{" separators=""><mml:mrow><mml:msqrt><mml:mrow><mml:mtext>ru</mml:mtext><mml:msup><mml:mrow><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo></mml:mrow><mml:mrow><mml:mn>2</mml:mn></mml:mrow></mml:msup><mml:mo>+</mml:mo><mml:mtext>mi</mml:mtext><mml:msup><mml:mrow><mml:mo>(</mml:mo><mml:mi>τ</mml:mi><mml:mo>)</mml:mo></mml:mrow><mml:mrow><mml:mn>2</mml:mn></mml:mrow></mml:msup></mml:mrow></mml:msqrt></mml:mrow></mml:mfenced><mml:mo>,</mml:mo></mml:mtd><mml:mtd><mml:mtext></mml:mtext></mml:mtd></mml:mtr></mml:mtable></mml:math><graphic xlink:href="13059_2016_1037_Article_Equb.gif" position="anchor"></graphic></alternatives></disp-formula></p><p>where ic(<italic>f</italic>) is the information content of the ontology term <italic>f</italic> [<xref ref-type="bibr" rid="CR10">10</xref>]. It is estimated in a maximum likelihood manner as the negative binary logarithm of the conditional probability that the term <italic>f</italic> is present in a protein’s annotation given that all its parent terms are also present. Note that here, <italic>n</italic><sub><italic>e</italic></sub>=<italic>n</italic> in the full evaluation mode and <italic>n</italic><sub><italic>e</italic></sub>=<italic>m</italic>(0) in the partial evaluation mode applies to both ru and mi.</p><p>In addition to the main metrics, we used two secondary metrics. Those were the weighted version of the precision–recall curves and the version of the remaining uncertainty–misinformation curves normalized to the [ 0,1] interval. These metrics and the corresponding evaluation results are shown in Additional file <xref rid="MOESM1" ref-type="media">1</xref>.</p><p>For the term-centric evaluation we used the area under the receiver operating characteristic (ROC) curve (AUC). The AUCs were calculated for all terms that have acquired at least ten positively annotated sequences, whereas the remaining benchmarks were used as negatives. The term-centric evaluation was used both for ranking models and to differentiate well and poorly predictable terms. The performance of each model on each term is provided in Additional file <xref rid="MOESM1" ref-type="media">1</xref>.</p><p>As we required all methods to keep two significant figures for prediction scores, the threshold <italic>τ</italic> in all metrics used in this study was varied from 0.01 to 1.00 with a step size of 0.01.</p></sec></sec><sec id="Sec8"><title>Data sets</title><p>Protein function annotations for the GO assessment were extracted, as a union, from three major protein databases that are available in the public domain: Swiss-Prot [<xref ref-type="bibr" rid="CR11">11</xref>], UniProt-GOA [<xref ref-type="bibr" rid="CR12">12</xref>] and the data from the GO consortium web site [<xref ref-type="bibr" rid="CR3">3</xref>]. We used evidence codes EXP, IDA, IPI, IMP, IGI, IEP, TAS, and IC to build benchmark and ground-truth sets. Annotations for the HPO assessment were downloaded from the HPO database [<xref ref-type="bibr" rid="CR8">8</xref>].</p><p>Figure <xref rid="Fig2" ref-type="fig">2</xref> summarizes the benchmarks we used in this study. Figure <xref rid="Fig2" ref-type="fig">2</xref><xref rid="Fig2" ref-type="fig">a</xref> shows the benchmark sizes for each of the ontologies and compares these numbers to CAFA1. All species that have at least 15 proteins in any of the benchmark categories are listed in Fig. <xref rid="Fig2" ref-type="fig">2</xref><xref rid="Fig2" ref-type="fig">b</xref>.
<fig id="Fig2"><label>Fig. 2</label><caption><p>CAFA2 benchmark breakdown. <bold>a</bold> The benchmark size for each of the four ontologies. <bold>b</bold> Breakdown of benchmarks for both types over 11 species (with no less than 15 proteins) sorted according to the total number of benchmark proteins. For both panels, <italic>dark colors</italic> (<italic>blue</italic>, <italic>red</italic>, and <italic>yellow</italic>) correspond to no-knowledge (NK) types, while their <italic>light color</italic> counterparts correspond to limited-knowledge (LK) types. The distributions of information contents corresponding to the benchmark sets are shown in Additional file <xref rid="MOESM1" ref-type="media">1</xref>. The size of CAFA 1 benchmarks are shown in <italic>gray</italic>. <italic>BPO</italic> Biological Process Ontology, <italic>CCO</italic> Cellular Component Ontology, <italic>HPO</italic> Human Phenotype Ontology, <italic>LK</italic> limited-knowledge, <italic>MFO</italic> Molecular Function Ontology, <italic>NK</italic> no-knowledge</p></caption><graphic xlink:href="13059_2016_1037_Fig2_HTML" id="MO2"></graphic></fig></p></sec><sec id="Sec9"><title>Comparison between CAFA1 and CAFA2 methods</title><p>We compared the results from CAFA1 and CAFA2 using a benchmark set that we created from CAFA1 targets and CAFA2 targets. More precisely, we used the stored predictions of the target proteins from CAFA1 and compared them with the new predictions from CAFA2 on the overlapping set of CAFA2 benchmarks and CAFA1 targets (a sequence had to be a no-knowledge target in both experiments to be eligible for this evaluation). For this analysis only, we used an artificial GO version by taking the intersection of the two GO snapshots (versions from January 2011 and June 2013) so as to mitigate the influence of ontology changes. We, thus, collected 357 benchmark proteins for MFO comparisons and 699 for BPO comparisons. The two baseline methods were trained on respective Swiss-Prot annotations for both ontologies so that they serve as controls for database change. In particular, SwissProt2011 (for CAFA1) contained 29,330 and 31,282 proteins for MFO and BPO, while SwissProt2014 (for CAFA2) contained 26,907 and 41,959 proteins for the two ontologies.</p><p>To conduct a head-to-head analysis between any two methods, we generated <italic>B</italic>=10,000 bootstrap samples and let methods compete on each such benchmark set. The performance improvement <italic>δ</italic> from CAFA1 to CAFA2 was calculated as
<disp-formula id="Equc"><alternatives><tex-math id="M7">\documentclass[12pt]{minimal}
\usepackage{amsmath}
\usepackage{wasysym}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{amsbsy}
\usepackage{mathrsfs}
\usepackage{upgreek}
\setlength{\oddsidemargin}{-69pt}
\begin{document} $$\begin{array}{@{}rcl@{}} \delta(m_{2}, m_{1}) = \frac{1}{B}\sum_{b=1}^{B} F_{\max}^{(b)}(m_{2}) - \frac{1}{B}\sum_{b=1}^{B} F_{\max}^{(b)}(m_{1}), \end{array} $$ \end{document}</tex-math><mml:math id="M8"><mml:mtable class="eqnarray" columnalign="left center right"><mml:mtr><mml:mtd class="eqnarray-1"><mml:mi>δ</mml:mi><mml:mo>(</mml:mo><mml:msub><mml:mrow><mml:mi>m</mml:mi></mml:mrow><mml:mrow><mml:mn>2</mml:mn></mml:mrow></mml:msub><mml:mo>,</mml:mo><mml:msub><mml:mrow><mml:mi>m</mml:mi></mml:mrow><mml:mrow><mml:mn>1</mml:mn></mml:mrow></mml:msub><mml:mo>)</mml:mo><mml:mo>=</mml:mo><mml:mfrac><mml:mrow><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:mi>B</mml:mi></mml:mrow></mml:mfrac><mml:munderover><mml:mrow><mml:mo>∑</mml:mo></mml:mrow><mml:mrow><mml:mi>b</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:mi>B</mml:mi></mml:mrow></mml:munderover><mml:msubsup><mml:mrow><mml:mi>F</mml:mi></mml:mrow><mml:mrow><mml:mo>max</mml:mo></mml:mrow><mml:mrow><mml:mo>(</mml:mo><mml:mi>b</mml:mi><mml:mo>)</mml:mo></mml:mrow></mml:msubsup><mml:mo>(</mml:mo><mml:msub><mml:mrow><mml:mi>m</mml:mi></mml:mrow><mml:mrow><mml:mn>2</mml:mn></mml:mrow></mml:msub><mml:mo>)</mml:mo><mml:mo>−</mml:mo><mml:mfrac><mml:mrow><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:mi>B</mml:mi></mml:mrow></mml:mfrac><mml:munderover><mml:mrow><mml:mo>∑</mml:mo></mml:mrow><mml:mrow><mml:mi>b</mml:mi><mml:mo>=</mml:mo><mml:mn>1</mml:mn></mml:mrow><mml:mrow><mml:mi>B</mml:mi></mml:mrow></mml:munderover><mml:msubsup><mml:mrow><mml:mi>F</mml:mi></mml:mrow><mml:mrow><mml:mo>max</mml:mo></mml:mrow><mml:mrow><mml:mo>(</mml:mo><mml:mi>b</mml:mi><mml:mo>)</mml:mo></mml:mrow></mml:msubsup><mml:mo>(</mml:mo><mml:msub><mml:mrow><mml:mi>m</mml:mi></mml:mrow><mml:mrow><mml:mn>1</mml:mn></mml:mrow></mml:msub><mml:mo>)</mml:mo><mml:mo>,</mml:mo></mml:mtd><mml:mtd class="eqnarray-2"></mml:mtd><mml:mtd class="eqnarray-3"></mml:mtd><mml:mtd><mml:mtext></mml:mtext></mml:mtd></mml:mtr></mml:mtable></mml:math><graphic xlink:href="13059_2016_1037_Article_Equc.gif" position="anchor"></graphic></alternatives></disp-formula></p><p>where <italic>m</italic><sub>1</sub> and <italic>m</italic><sub>2</sub> stand for methods from CAFA1 and CAFA2, respectively, and <inline-formula id="IEq2"><alternatives><tex-math id="M9">\documentclass[12pt]{minimal}
\usepackage{amsmath}
\usepackage{wasysym}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{amsbsy}
\usepackage{mathrsfs}
\usepackage{upgreek}
\setlength{\oddsidemargin}{-69pt}
\begin{document}$F_{\max }^{(b)}(\cdot)$\end{document}</tex-math><mml:math id="M10"><mml:msubsup><mml:mrow><mml:mi>F</mml:mi></mml:mrow><mml:mrow><mml:mo>max</mml:mo></mml:mrow><mml:mrow><mml:mo>(</mml:mo><mml:mi>b</mml:mi><mml:mo>)</mml:mo></mml:mrow></mml:msubsup><mml:mo>(</mml:mo><mml:mo>·</mml:mo><mml:mo>)</mml:mo></mml:math><inline-graphic xlink:href="13059_2016_1037_Article_IEq2.gif"></inline-graphic></alternatives></inline-formula> represents the <italic>F</italic><sub>max</sub> of a method evaluated on the <italic>b</italic>-th bootstrapped benchmark set.</p></sec><sec id="Sec10"><title>Baseline models</title><p>We built two baseline methods, Naïve and BLAST, and compared them with all participating methods. The Naïve method simply predicts the frequency of a term being annotated in a database [<xref ref-type="bibr" rid="CR13">13</xref>]. BLAST was based on search results using the Basic Local Alignment Search Tool (BLAST) software against the training database [<xref ref-type="bibr" rid="CR14">14</xref>]. A term will be predicted as the highest local alignment sequence identity among all BLAST hits annotated with the term. Both of these methods were trained on the experimentally annotated proteins available in Swiss-Prot at time <italic>t</italic><sub>0</sub>, except for HPO where the two baseline models were trained using the annotations from the <italic>t</italic><sub>0</sub> release of the HPO.</p></sec></sec><sec id="Sec11"><title>Results and discussion</title><sec id="Sec12"><title>Top methods have improved since CAFA1</title><p>We conducted the second CAFA experiment 3 years after the first one. As our knowledge of protein function has increased since then, it was worthwhile to assess whether computational methods have also been improved and if so, to what extent. Therefore, to monitor the progress over time, we revisit some of the top methods in CAFA1 and compare them with their successors.</p><p>For each benchmark set we carried out a bootstrap-based comparison between a pair of top-ranked methods (one from CAFA1 and another from CAFA2), as described in “<xref rid="Sec2" ref-type="sec">Methods</xref>”. The average performance metric as well as the number of wins were recorded (in the case of identical performance, neither method was awarded a win). Figure <xref rid="Fig3" ref-type="fig">3</xref> summarizes the results of this analysis. We use a color code from orange to blue to indicate the performance improvement <italic>δ</italic> from CAFA1 to CAFA2.
<fig id="Fig3"><label>Fig. 3</label><caption><p>CAFA1 versus CAFA2 (<italic>top methods</italic>). A comparison in <italic>F</italic><sub>max</sub> between the top-five CAFA1 models against the top-five CAFA2 models. <italic>Colored boxes</italic> encode the results such that (1) the colors indicate margins of a CAFA2 method over a CAFA1 method in <italic>F</italic><sub>max</sub> and (2) the numbers in the <italic>box</italic> indicate the percentage of wins. For both the Molecular Function Ontology (<bold>a</bold>) and Biological Process Ontology (<bold>b</bold>) results: <bold>A</bold> CAFA1 top-five models (<italic>rows, from top to bottom</italic>) against CAFA2 top-five models (<italic>columns, from left to right</italic>). <bold>B</bold> Comparison of Naïve baselines trained respectively on SwissProt2011 and SwissProt2014. <bold>C</bold> Comparison of BLAST baselines trained on SwissProt2011 and SwissProt2014</p></caption><graphic xlink:href="13059_2016_1037_Fig3_HTML" id="MO3"></graphic></fig></p><p>The selection of top methods for this study was based on their performance in each ontology on the entire benchmark sets. Panels B and C in Fig. <xref rid="Fig3" ref-type="fig">3</xref> compare baseline methods trained on different data sets. We see no improvements of these baselines except for BLAST on BPO where it is slightly better to use the newer version of Swiss-Prot as the reference database for the search. On the other hand, all top methods in CAFA2 outperformed their counterparts in CAFA1. For predicting molecular functions, even though transferring functions from BLAST hits does not give better results, the top models still managed to perform better. It is possible that the newly acquired annotations since CAFA1 enhanced BLAST, which involves direct function transfer, and perhaps lead to better performances of those downstream methods that rely on sequence alignments. However, this effect does not completely explain the extent of the performance improvement achieved by those methods. This is promising evidence that top methods from the community have improved since CAFA1 and that improvements were not simply due to updates of curated databases.</p></sec><sec id="Sec13"><title>Protein-centric evaluation</title><p>Protein-centric evaluation measures how accurately methods can assign functional terms to a protein. The protein-centric performance evaluation of the top-ten methods is shown in Figs. <xref rid="Fig4" ref-type="fig">4</xref>, <xref rid="Fig5" ref-type="fig">5</xref>, and <xref rid="Fig6" ref-type="fig">6</xref>. The 95 % confidence intervals were estimated using bootstrapping on the benchmark set with <italic>B</italic>=10,000 iterations [<xref ref-type="bibr" rid="CR15">15</xref>]. The results provide a broad insight into the state of the art.
<fig id="Fig4"><label>Fig. 4</label><caption><p>Overall evaluation using the maximum <italic>F</italic> measure, <italic>F</italic><sub>max</sub>. Evaluation was carried out on no-knowledge benchmark sequences in the full mode. The coverage of each method is shown within its performance bar. A perfect predictor would be characterized with <italic>F</italic><sub>max</sub>=1. Confidence intervals (95 %) were determined using bootstrapping with 10,000 iterations on the set of benchmark sequences. For cases in which a principal investigator participated in multiple teams, the results of only the best-scoring method are presented. Details for all methods are provided in Additional file <xref rid="MOESM1" ref-type="media">1</xref></p></caption><graphic xlink:href="13059_2016_1037_Fig4_HTML" id="MO4"></graphic></fig><fig id="Fig5"><label>Fig. 5</label><caption><p>Precision–recall curves for top-performing methods. Evaluation was carried out on no-knowledge benchmark sequences in the full mode. A perfect predictor would be characterized with <italic>F</italic><sub>max</sub>=1, which corresponds to the point (1,1) in the precision–recall plane. For cases in which a principal investigator participated in multiple teams, the results of only the best-scoring method are presented</p></caption><graphic xlink:href="13059_2016_1037_Fig5_HTML" id="MO5"></graphic></fig><fig id="Fig6"><label>Fig. 6</label><caption><p>Overall evaluation using the minimum semantic distance, <italic>S</italic><sub>min</sub>. Evaluation was carried out on no-knowledge benchmark sequences in the full mode. The coverage of each method is shown within its performance bar. A perfect predictor would be characterized with <italic>S</italic><sub>min</sub>=0. Confidence intervals (95 %) were determined using bootstrapping with 10,000 iterations on the set of benchmark sequences. For cases in which a principal investigator participated in multiple teams, the results of only the best-scoring method are presented. Details for all methods are provided in Additional file <xref rid="MOESM1" ref-type="media">1</xref></p></caption><graphic xlink:href="13059_2016_1037_Fig6_HTML" id="MO6"></graphic></fig></p><p>Predictors performed very differently across the four ontologies. Various reasons contribute to this effect including: (1) the topological properties of the ontology such as the size, depth, and branching factor; (2) term predictability; for example, the BPO terms are considered to be more abstract in nature than the MFO and CCO terms; (3) the annotation status, such as the size of the training set at <italic>t</italic><sub>0</sub>, the annotation depth of benchmark proteins, as well as various annotation biases [<xref ref-type="bibr" rid="CR6">6</xref>].</p><p>In general, CAFA2 methods perform better at predicting MFO terms than any other ontology. Top methods achieved <italic>F</italic><sub>max</sub> scores around 0.6 and considerably surpassed the two baseline models. Maintaining the pattern from CAFA1, the performance accuracies in the BPO category were not as good as in the MFO category. The best-performing method scored slightly below 0.4.</p><p>For the two newly added ontologies in CAFA2, we observed that the top predictors performed no better than the Naïve method under <italic>F</italic><sub>max</sub>, whereas they slightly outperformed the Naïve method under <italic>S</italic><sub>min</sub> in CCO. One reason for the competitive performance of the Naïve method in the CCO category is that a small number of relatively general terms are frequently used, and those relative frequencies do not diffuse quickly enough with the depth of the graph. For instance, the annotation frequency of “organelle” (GO:0043226, level 2), “intracellular part” (GO:0044424, level 3), and “cytoplasm” (GO:0005737, level 4) are all above the best threshold for the Naïve method (<italic>τ</italic><sub>optimal</sub>=0.32). Correctly predicting these terms increases the number of true positives and thus boosts the performance of the Naïve method under the <italic>F</italic><sub>max</sub> evaluation. However, once the less informative terms are down-weighted (using the <italic>S</italic><sub>min</sub> measure), the Naïve method becomes significantly penalized and degraded. Another reason for the comparatively good performance of Naïve is that the benchmark proteins were annotated with more general terms than the (training) proteins previously deposited in the UniProt database. This effect was most prominent in the CCO (Additional file <xref rid="MOESM1" ref-type="media">1</xref>: Figure S2) and has thus artificially boosted the performance of the Naïve method. The weighted <italic>F</italic><sub>max</sub> and normalized <italic>S</italic><sub>min</sub> evaluations can be found in Additional file <xref rid="MOESM1" ref-type="media">1</xref>.</p><p>Interestingly, generally shallower annotations of benchmark proteins do not seem to be the major reason for the observed performance in the HPO category. One possibility for the observed performance is that, unlike for GO terms, the HPO annotations are difficult to transfer from other species. Another possibility is the sparsity of experimental annotations. The current number of experimentally annotated proteins in HPO is 4794, i.e., 0.5 proteins per HPO term, which is at least an order of magnitude less than for other ontologies. Finally, the relatively high frequency of general terms may have also contributed to the good performance of Naïve. We originally hypothesized that a possible additional explanation for this effect might be that the average number of HPO terms associated with a human protein is considerably larger than in GO; i.e., the mean number of annotations per protein in HPO is 84, while for MFO, BPO, and CCO, the mean number of annotations per protein is 10, 39, and 14, respectively. However, we do not observe this effect in other ontologies when the benchmark proteins are split into those with a low or high number of terms. Overall, successfully predicting the HPO terms in the protein-centric mode is a difficult problem and further effort will be required to fully characterize the performance.</p></sec><sec id="Sec14"><title>Term-centric evaluation</title><p>The protein-centric view, despite its power in showing the strengths of a predictor, does not gauge a predictor’s performance for a specific function. In a term-centric evaluation, we assess the ability of each method to identify new proteins that have a particular function, participate in a process, are localized to a component, or affect a human phenotype. To assess this term-wise accuracy, we calculated AUCs in the prediction of individual terms. Averaging the AUC values over terms provides a metric for ranking predictors, whereas averaging predictor performance over terms provides insights into how well this term can be predicted computationally by the community.</p><p>Figure <xref rid="Fig7" ref-type="fig">7</xref> shows the performance evaluation where the AUCs for each method were averaged over all terms for which at least ten positive sequences were available. Proteins without predictions were counted as predictions with a score of 0. As shown in Figs. <xref rid="Fig4" ref-type="fig">4</xref>, <xref rid="Fig5" ref-type="fig">5</xref>, and <xref rid="Fig6" ref-type="fig">6</xref>, correctly predicting CCO and HPO terms for a protein might not be an easy task according to the protein-centric results. However, the overall poor performance could also result from the dominance of poorly predictable terms. Therefore, a term-centric view can help differentiate prediction quality across terms. As shown in Fig. <xref rid="Fig8" ref-type="fig">8</xref>, most of the terms in HPO obtain an AUC greater than the Naïve model, with some terms on average achieving reasonably well AUCs around 0.7. Depending on the training data available for participating methods, well-predicted phenotype terms range from mildly specific such as “Lymphadenopathy” and “Thrombophlebitis” to general ones such as “Abnormality of the Skin Physiology”.
<fig id="Fig7"><label>Fig. 7</label><caption><p>Overall evaluation using the averaged AUC over terms with no less than ten positive annotations. The evaluation was carried out on no-knowledge benchmark sequences in the full mode. Error bars indicate the standard error in averaging AUC over terms for each method. For cases in which a principal investigator participated in multiple teams, the results of only the best-scoring method are presented. Details for all methods are provided in Additional file <xref rid="MOESM1" ref-type="media">1</xref>. <italic>AUC</italic> receiver operating characteristic curve</p></caption><graphic xlink:href="13059_2016_1037_Fig7_HTML" id="MO7"></graphic></fig><fig id="Fig8"><label>Fig. 8</label><caption><p>Averaged AUC per term for Human Phenotype Ontology. <bold>a</bold> Terms are sorted based on AUC. The <italic>dashed red line</italic> indicates the performance of the Naïve method. <bold>b</bold> The top-ten accurately predicted terms without overlapping ancestors (except for the root). <italic>AUC</italic> receiver operating characteristic curve</p></caption><graphic xlink:href="13059_2016_1037_Fig8_HTML" id="MO8"></graphic></fig></p></sec><sec id="Sec15"><title>Performance on various categories of benchmarks</title><sec id="Sec16"><title>Easy versus difficult benchmarks</title><p>As in CAFA1, the no-knowledge GO benchmarks were divided into easy versus difficult categories based on their maximal global sequence identity with proteins in the training set. Since the distribution of sequence identities roughly forms a bimodal shape (Additional file <xref rid="MOESM1" ref-type="media">1</xref>), a cutoff of 60 % was manually chosen to define the two categories. The same cutoff was used in CAFA1. Unsurprisingly, across all three ontologies, the performance of the BLAST model was substantially impacted for the difficult category because of the lack of high sequence identity homologs and as a result, transferring annotations was relatively unreliable. However, we also observed that most top methods were insensitive to the types of benchmarks, which provides us with encouraging evidence that state-of-the-art protein function predictors can successfully combine multiple potentially unreliable hits, as well as multiple types of data, into a reliable prediction.</p></sec><sec id="Sec17"><title>Species-specific categories</title><p>The benchmark proteins were split into even smaller categories for each species as long as the resulting category contained at least 15 sequences. However, because of space limitations, in Fig. <xref rid="Fig9" ref-type="fig">9</xref> we show the breakdown results on only eukarya and prokarya benchmarks; the species-specific results are provided in Additional file <xref rid="MOESM1" ref-type="media">1</xref>. It is worth noting that the performance accuracies on the entire benchmark sets were dominated by the targets from eukarya due to their larger proportion in the benchmark set and annotation preferences. The eukarya benchmark rankings therefore coincide with the overall rankings, but the smaller categories typically showed different rankings and may be informative to more specialized research groups.
<fig id="Fig9"><label>Fig. 9</label><caption><p>Performance evaluation using the maximum <italic>F</italic> measure, <italic>F</italic><sub>max</sub>, on eukaryotic (<italic>left</italic>) versus prokaryotic (<italic>right</italic>) benchmark sequences. The evaluation was carried out on no-knowledge benchmark sequences in the full mode. The coverage of each method is shown within its performance bar. Confidence intervals (95 %) were determined using bootstrapping with 10,000 iterations on the set of benchmark sequences. For cases in which a principal investigator participated in multiple teams, the results of only the best-scoring method are presented. Details for all methods are provided in Additional file <xref rid="MOESM1" ref-type="media">1</xref></p></caption><graphic xlink:href="13059_2016_1037_Fig9_HTML" id="MO9"></graphic></fig></p><p>For all three GO ontologies, no-knowledge prokarya benchmark sequences collected over the annotation growth phase mostly (over 80 %) came from two species: <italic>Escherichia coli</italic> and <italic>Pseudomonas aeruginosa</italic> (for CCO, 21 out of 22 proteins were from <italic>E. coli</italic>). Thus, one should keep in mind that the prokarya benchmarks essentially reflect the performance on proteins from these two species. Methods predicting the MFO terms for prokaryotes are slightly worse than those for eukaryotes. In addition, direct function transfer by homology for prokaryotes did not work well using this ontology. However, the performance was better using the other two ontologies, especially CCO. It is not very surprising that the top methods achieved good performance for <italic>E. coli</italic> as it is a well-studied model organism.</p></sec></sec><sec id="Sec18"><title>Diversity of predictions</title><p>Evaluation of the top methods revealed that performance was often statistically indistinguishable between the best methods. This could result from all top methods making the same predictions, or from different prediction sets resulting in the same summarized performance. To assess this, we analyzed the extent to which methods generated similar predictions within each ontology. Specifically, we calculated the pairwise Pearson correlation between methods on a common set of gene-concept pairs and then visualized these similarities as networks (for BPO, see Fig. <xref rid="Fig10" ref-type="fig">10</xref>; for MFO, CCO, and HPO, see Additional file <xref rid="MOESM1" ref-type="media">1</xref>).
<fig id="Fig10"><label>Fig. 10</label><caption><p>Similarity network of participating methods for BPO. Similarities are computed as Pearson’s correlation coefficient between methods, with a 0.75 cutoff for illustration purposes. A unique color is assigned to all methods submitted under the same principal investigator. Not evaluated (organizers’) methods are shown in <italic>triangles</italic>, while benchmark methods (Naïve and BLAST) are shown in <italic>squares</italic>. The top-ten methods are highlighted with enlarged nodes and <italic>circled in red</italic>. The edge width indicates the strength of similarity. Nodes are labeled with the name of the methods followed by “-team(model)” if multiple teams/models were submitted</p></caption><graphic xlink:href="13059_2016_1037_Fig10_HTML" id="MO10"></graphic></fig></p><p>In MFO, where we observed the highest overall performance of prediction methods, eight of the ten top methods were in the largest connected component. In addition, we observed a high connectivity between methods, suggesting that the participating methods are leveraging similar sources of data in similar ways. Predictions for BPO showed a contrasting pattern. In this ontology, the largest connected component contained only two of the top-ten methods. The other top methods were contained in components made up of other methods produced by the same lab. This suggests that the approaches that participating groups have taken generate more diverse predictions for this ontology and that there are many different paths to a top-performing biological process prediction method. Results for HPO were more similar to those for BPO, while results for cellular component were more similar in structure to molecular function.</p><p>Taken together, these results suggest that ensemble approaches that aim to include independent sources of high-quality predictions may benefit from leveraging the data and techniques used by different research groups and that such approaches that effectively weigh and integrate disparate methods may demonstrate more substantial improvements over existing methods in the process and phenotype ontologies where current prediction approaches share less similarity.</p><p>At the time that authors submitted predictions, we also asked them to select from a list of 30 keywords that best describe their methodology. We examined these author-assigned keywords for methods that ranked in the top ten to determine what approaches were used in currently high-performing methods (Additional file <xref rid="MOESM1" ref-type="media">1</xref>). Sequence alignment and machine-learning methods were in the top-three terms for all ontologies. For biological process, the other member of the top three is protein–protein interactions, while for cellular component and molecular function the third member is sequence properties. The broad sets of keywords among top-performing methods further suggest that these methods are diverse in their inputs and approach.</p></sec><sec id="Sec19"><title>Case study: ADAM-TS12</title><p>To illustrate some of the challenges and accomplishments of CAFA, we provide an in-depth examination of the prediction of the functional terms of one protein, human ADAM-TS12 [<xref ref-type="bibr" rid="CR16">16</xref>]. ADAMs (a disintegrin and metalloproteinase) are a family of secreted metallopeptidases featuring a pro-domain, a metalloproteinase, a disintegrin, a cysteine-rich epidermal growth-factor-like domain, and a transmembrane domain [<xref ref-type="bibr" rid="CR17">17</xref>]. The ADAM-TS subfamily include eight thrombospondin type-1 (TS-1) motifs; it is believed to play a role in fetal pulmonary development and may have a role as a tumor suppressor, specifically the negative regulation of the hepatocyte growth factor receptor signaling pathway [<xref ref-type="bibr" rid="CR18">18</xref>].</p><p>We did not observe any experimental annotation by the time submission was closed. Annotations were later deposited to all three GO ontologies during the growth phase of CAFA2. Therefore, ADAM-TS12 was considered a no-knowledge benchmark protein for our assessment in all GO ontologies. The total number of leaf terms to predict for biological process was 12; these nodes induced a directed acyclic annotation graph consisting of 89 nodes. In Fig. <xref rid="Fig11" ref-type="fig">11</xref> we show the performance of the top-five methods in predicting the BPO terms that are experimentally verified to be associated with ADAM-TS12.
<fig id="Fig11"><label>Fig. 11</label><caption><p>Case study on the human <italic>ADAM-TS12</italic> gene. Biological process terms associated with <italic>ADAM-TS12</italic> gene in the union of the three databases by September 2014. The entire functional annotation of <italic>ADAM-TS12</italic> consists of 89 terms, 28 of which are shown. Twelve terms, marked in <italic>green</italic>, are leaf terms. This directed acyclic graph was treated as ground truth in the CAFA2 assessment. <italic>Solid black lines</italic> provide direct “is a” or “part of” relationships between terms, while <italic>gray lines</italic> mark indirect relationships (that is, some terms were not drawn in this picture). Predicted terms of the top-five methods and two baseline methods were picked at their optimal <italic>F</italic><sub>max</sub> threshold. Over-predicted terms are not shown</p></caption><graphic xlink:href="13059_2016_1037_Fig11_HTML" id="MO11"></graphic></fig></p><p>As can be seen, most methods correctly discovered non-leaf nodes with a moderate amount of information content. “Glycoprotein Catabolic Process”, “Cellular Response to Stimulus”, and “Proteolysis” were the best discovered GO terms by the top-five performers. The Paccanaro Lab (P) discovered several additional correct leaf terms. It is interesting to note that only BLAST successfully predicted “Negative regulation of signal transduction” whereas the other methods did not. The reason for this is that we set the threshold for reporting a discovery when the confidence score for a term was equal to or exceeded the method’s <italic>F</italic><sub>max</sub>. In this particular case, the Paccanaro Lab method did predict the term, but the confidence score was 0.01 below their <italic>F</italic><sub>max</sub> threshold.</p><p>This example illustrates both the success and the difficulty of correctly predicting highly specific terms in BPO, especially with a protein that is involved in four distinct cellular processes: in this case, regulation of cellular growth, proteolysis, cellular response to various cytokines, and cell-matrix adhesion. Additionally, this example shows that the choices that need to be made when assessing method performance may cause some loss of information with respect to the method’s actual performance. That is, the way we capture a method’s performance in CAFA may not be exactly the same as a user may employ. In this case, a user may choose to include lower confidence scores when running the Paccanaro Lab method, and include the term “Negative regulation of signal transduction” in the list of accepted predictions.</p></sec></sec><sec id="Sec20" sec-type="conclusion"><title>Conclusions</title><p>Accurately annotating the function of biological macromolecules is difficult, and requires the concerted effort of experimental scientists, biocurators, and computational biologists. Though challenging, advances are valuable: accurate predictions allow biologists to rapidly generate testable hypotheses about how proteins fit into processes and pathways. We conducted the second CAFA challenge to assess the status of the computational function prediction of proteins and to quantify the progress in the field.</p><sec id="Sec21"><title>The field has moved forward</title><p>Three years ago, in CAFA1, we concluded that the top methods for function prediction outperform straightforward function transfer by homology. In CAFA2, we observe that the methods for function prediction have improved compared to those from CAFA1. As part of the CAFA1 experiment, we stored all predictions from all methods on 48,298 target proteins from 18 species. We compared those stored predictions to the newly deposited predictions from CAFA2 on the overlapping set of benchmark proteins and CAFA1 targets. The head-to-head comparisons among the top-five CAFA1 methods against the top-five CAFA2 methods reveal that the top CAFA2 methods outperformed all top CAFA1 methods.</p><p>Our parallel evaluation using an unchanged BLAST algorithm with data from 2011 and data from 2014 showed little difference, strongly suggesting that the improvements observed are due to methodological advances. The lessons from CAFA1 and annual AFP-SIG during the ISMB conference, where new developments are rapidly disseminated, may have contributed to this outcome [<xref ref-type="bibr" rid="CR19">19</xref>].</p></sec><sec id="Sec22"><title>Evaluation metrics</title><p>A universal performance assessment in protein function prediction is far from straightforward. Although various evaluation metrics have been proposed under the framework of multi-label and structured-output learning, the evaluation in this subfield also needs to be interpretable to a broad community of researchers as well as the public. To address this, we used several metrics in this study as each provides useful insights and complements the others. Understanding the strengths and weaknesses of current metrics and developing better metrics remain important.</p><p>One important observation with respect to metrics is that the protein-centric and term-centric views may give different perspectives to the same problem. For example, while in MFO and BPO we generally observe a positive correlation between the two, in CCO and HPO these different metrics may lead to entirely different interpretations of an experiment. Regardless of the underlying cause, as discussed in “<xref rid="Sec11" ref-type="sec">Results and discussion</xref>”, it is clear that some ontological terms are predictable with high accuracy and can be reliably used in practice even in these ontologies. In the meantime, more effort will be needed to understand the problems associated with the statistical and computational aspects of method development.</p></sec><sec id="Sec23"><title>Well-performing methods</title><p>We observe that participating methods usually specialize in one or few categories of protein function prediction, and have been developed with their own application objectives in mind. Therefore, the performance rankings of methods often change from one benchmark set to another. There are complex factors that influence the final ranking including the selection of the ontology, types of benchmark sets and evaluation, as well as evaluation metrics, as discussed earlier. Most of our assessment results show that the performances of top-performing methods are generally comparable to each other. It is worth noting that performance is usually better in predicting molecular function than other ontologies.</p><p>Beyond simply showing diversity in inputs, our evaluation of prediction similarity revealed that many top-performing methods are reaching this status by generating distinct predictions, suggesting that there is additional room for continued performance improvement. Although a small group of methods could be considered as generally high performing, there is no single method that dominates over all benchmarks. Taken together, these results highlight the potential for ensemble learning approaches in this domain.</p><p>We also observed that when provided with a chance to select a reliable set of predictions, the methods generally perform better (partial evaluation mode versus full evaluation mode). This outcome is encouraging; it suggests that method developers can predict where their methods are particularly accurate and target them to that space.</p><p>Our keyword analysis showed that machine-learning methods are widely used by successful approaches. Protein interactions were more overrepresented in the best-performing methods for biological process prediction. This suggests that predicting membership in pathways and processes requires information on interacting partners in addition to a protein’s sequence features.</p></sec><sec id="Sec24"><title>Final notes</title><p>Automated functional annotation remains an exciting and challenging task, central to understanding genomic data, which are central to biomedical research. Three years after CAFA1, the top methods from the community have shown encouraging progress. However, in terms of raw scores, there is still significant room for improvement in all ontologies, and particularly in BPO, CCO, and HPO. There is also a need to develop an experiment-driven, as opposed to curation-driven, component of the evaluation to address limitations for term-centric evaluation. In the future CAFA experiments, we will continue to monitor the performance over time and invite a broad range of computational biologists, computer scientists, statisticians, and others to address these engaging problems of concept annotation for biological macromolecules through CAFA.</p><p>CAFA2 significantly expanded the number of protein targets, the number of biomedical ontologies used for annotation, the number of analysis scenarios, as well as the metrics used for evaluation. The results of the CAFA2 experiment detail the state of the art in protein function prediction, can guide the development of new concept annotation methods, and help molecular biologists assess the relative reliability of predictions. Understanding the function of biological macromolecules brings us closer to understanding life at the molecular level and improving human health.</p></sec></sec></body><back><app-group><app id="App1"><sec id="Sec25"><title>Additional file</title><p><media position="anchor" xlink:href="13059_2016_1037_MOESM1_ESM.pdf" id="MOESM1"><label>Additional file 1</label><caption><p>A document containing a subset of CAFA2 analyses that are equivalent to those provided about the CAFA1 experiment in the CAFA1 supplement. (PDF 11100 kb)</p></caption></media></p></sec></app></app-group><ack><sec id="d30e3714"><title>Funding</title><p>We acknowledge the contributions of Maximilian Hecht, Alexander Grün, Julia Krumhoff, My Nguyen Ly, Jonathan Boidol, Rene Schoeffel, Yann Spöri, Jessika Binder, Christoph Hamm and Karolina Worf. This work was partially supported by the following grants: National Science Foundation grants DBI-1458477 (PR), DBI-1458443 (SDM), DBI-1458390 (CSG), DBI-1458359 (IF), IIS-1319551 (DK), DBI-1262189 (DK), and DBI-1149224 (JC); National Institutes of Health grants R01GM093123 (JC), R01GM097528 (DK), R01GM076990 (PP), R01GM071749 (SEB), R01LM009722 (SDM), and UL1TR000423 (SDM); the National Natural Science Foundation of China grants 3147124 (WT) and 91231116 (WT); the National Basic Research Program of China grant 2012CB316505 (WT); NSERC grant RGPIN 371348-11 (PP); FP7 infrastructure project TransPLANT Award 283496 (ADJvD); Microsoft Research/FAPESP grant 2009/53161-6 and FAPESP fellowship 2010/50491-1 (DCAeS); Biotechnology and Biological Sciences Research Council grants BB/L020505/1 (DTJ), BB/F020481/1 (MJES), BB/K004131/1 (AP), BB/F00964X/1 (AP), and BB/L018241/1 (CD); the Spanish Ministry of Economics and Competitiveness grant BIO2012-40205 (MT); KU Leuven CoE PFV/10/016 SymBioSys (YM); the Newton International Fellowship Scheme of the Royal Society grant NF080750 (TN). CSG was supported in part by the Gordon and Betty Moore Foundation’s Data-Driven Discovery Initiative grant GBMF4552. Computational resources were provided by CSC – IT Center for Science Ltd., Espoo, Finland (TS). This work was supported by the Academy of Finland (TS). RCL and ANM were supported by British Heart Foundation grant RG/13/5/30112. PD, RCL, and REF were supported by Parkinson’s UK grant G-1307, the Alexander von Humboldt Foundation through the German Federal Ministry for Education and Research, Ernst Ludwig Ehrlich Studienwerk, and the Ministry of Education, Science and Technological Development of the Republic of Serbia grant 173001. This work was a Technology Development effort for ENIGMA – Ecosystems and Networks Integrated with Genes and Molecular Assemblies (<ext-link ext-link-type="uri" xlink:href="http://enigma.lbl.gov">http://enigma.lbl.gov</ext-link>), a Scientific Focus Area Program at Lawrence Berkeley National Laboratory, which is based upon work supported by the U.S. Department of Energy, Office of Science, Office of Biological & Environmental Research grant DE-AC02-05CH11231. ENIGMA only covers the application of this work to microbial proteins. NSF DBI-0965616 and Australian Research Council grant DP150101550 (KMV). NSF DBI-0965768 (ABH). NIH T15 LM00945102 (training grant for CSF). FP7 FET grant MAESTRA ICT-2013-612944 and FP7 REGPOT grant InnoMol (FS). NIH R01 GM60595 (PCB). University of Padova grants CPDA138081/13 (ST) and GRIC13AAI9 (EL). Swiss National Science Foundation grant 150654 and UK BBSRC grant BB/M015009/1 (COD). PRB2 IPT13/0001 - ISCIII-SGEFI / FEDER (JMF).</p></sec><sec id="d30e3724"><title>Availability of data and materials</title><p><bold>Data</bold> The benchmark data and the predictions are available on FigShare 10.6084/m9.figshare.2059944.v1. Note that according to CAFA rules, all but the top-ten methods are anonymized. However, methods are uniquely identified by a code number, so use of the data for further analysis is possible.</p><p><bold>Software</bold> The code used in this study is available at <ext-link ext-link-type="uri" xlink:href="https://github.com/yuxjiang/CAFA2">https://github.com/yuxjiang/CAFA2</ext-link>.</p></sec><sec id="d30e3745"><title>Authors’ contributions</title><p>PR and IF conceived of the CAFA experiment and supervised the project. YJ performed most analyses and significantly contributed to the writing. PR, IF, and CSG significantly contributed to writing the manuscript. IF, PR, CSG, WTC, ARB, DD, and RL contributed to the analyses. SDM managed the data acquisition. TRO developed the web interface, including the portal for submission and the storage of predictions. RPH, MJM, and CO’D directed the biocuration efforts. EC-U, PD, REF, RH, DL, RCL, MM, ANM, PM-M, KP, and AS performed the biocuration. YM and PNR co-organized the human phenotype challenge. ML, AT, PCB, SEB, CO, and BR steered the CAFA experiment and provided critical guidance. The remaining authors participated in the experiment, provided writing and data for their methods, and contributed comments on the manuscript. All authors read and approved the final manuscript.</p></sec><sec id="d30e3750"><title>Competing interests</title><p>The authors declare that they have no competing interests.</p></sec><sec id="d30e3755"><title>Ethics approval and consent to participate</title><p>Not applicable to this work.</p></sec></ack><ref-list id="Bib1"><title>References</title><ref id="CR1"><label>1</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Costello</surname><given-names>JC</given-names></name><name><surname>Stolovitzky</surname><given-names>G</given-names></name></person-group><article-title>Seeking the wisdom of crowds through challenge-based competitions in biomedical research</article-title><source>Clin Pharmacol Ther</source><year>2013</year><volume>93</volume><issue>5</issue><fpage>396</fpage><lpage>8</lpage><pub-id pub-id-type="doi">10.1038/clpt.2013.36</pub-id><pub-id pub-id-type="pmid">23549146</pub-id></element-citation></ref><ref id="CR2"><label>2</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Radivojac</surname><given-names>P</given-names></name><name><surname>Clark</surname><given-names>WT</given-names></name><name><surname>Oron</surname><given-names>TR</given-names></name><name><surname>Schnoes</surname><given-names>AM</given-names></name><name><surname>Wittkop</surname><given-names>T</given-names></name><name><surname>Sokolov</surname><given-names>A</given-names></name><etal></etal></person-group><article-title>A large-scale evaluation of computational protein function prediction</article-title><source>Nat Methods</source><year>2013</year><volume>10</volume><issue>3</issue><fpage>221</fpage><lpage>7</lpage><pub-id pub-id-type="doi">10.1038/nmeth.2340</pub-id><pub-id pub-id-type="pmid">23353650</pub-id></element-citation></ref><ref id="CR3"><label>3</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Ashburner</surname><given-names>M</given-names></name><name><surname>Ball</surname><given-names>CA</given-names></name><name><surname>Blake</surname><given-names>JA</given-names></name><name><surname>Botstein</surname><given-names>D</given-names></name><name><surname>Butler</surname><given-names>H</given-names></name><name><surname>Cherry</surname><given-names>JM</given-names></name><etal></etal></person-group><article-title>Gene Ontology: tool for the unification of biology</article-title><source>Nat Genet</source><year>2000</year><volume>25</volume><issue>1</issue><fpage>25</fpage><lpage>9</lpage><pub-id pub-id-type="doi">10.1038/75556</pub-id><pub-id pub-id-type="pmid">10802651</pub-id></element-citation></ref><ref id="CR4"><label>4</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Dessimoz</surname><given-names>C</given-names></name><name><surname>Skunca</surname><given-names>N</given-names></name><name><surname>Thomas</surname><given-names>PD</given-names></name></person-group><article-title>CAFA and the open world of protein function predictions</article-title><source>Trends Genet</source><year>2013</year><volume>29</volume><issue>11</issue><fpage>609</fpage><lpage>10</lpage><pub-id pub-id-type="doi">10.1016/j.tig.2013.09.005</pub-id><pub-id pub-id-type="pmid">24138813</pub-id></element-citation></ref><ref id="CR5"><label>5</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Gillis</surname><given-names>J</given-names></name><name><surname>Pavlidis</surname><given-names>P</given-names></name></person-group><article-title>Characterizing the state of the art in the computational assignment of gene function: lessons from the first critical assessment of functional annotation (CAFA)</article-title><source>BMC Bioinform</source><year>2013</year><volume>14</volume><issue>Suppl 3</issue><fpage>15</fpage><pub-id pub-id-type="doi">10.1186/1471-2105-14-S3-S15</pub-id></element-citation></ref><ref id="CR6"><label>6</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Schnoes</surname><given-names>AM</given-names></name><name><surname>Ream</surname><given-names>DC</given-names></name><name><surname>Thorman</surname><given-names>AW</given-names></name><name><surname>Babbitt</surname><given-names>PC</given-names></name><name><surname>Friedberg</surname><given-names>I</given-names></name></person-group><article-title>Biases in the experimental annotations of protein function and their effect on our understanding of protein function space</article-title><source>PLoS Comput Biol</source><year>2013</year><volume>9</volume><issue>5</issue><fpage>1003063</fpage><pub-id pub-id-type="doi">10.1371/journal.pcbi.1003063</pub-id></element-citation></ref><ref id="CR7"><label>7</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Jiang</surname><given-names>Y</given-names></name><name><surname>Clark</surname><given-names>WT</given-names></name><name><surname>Friedberg</surname><given-names>I</given-names></name><name><surname>Radivojac</surname><given-names>P</given-names></name></person-group><article-title>The impact of incomplete knowledge on the evaluation of protein function prediction: a structured-output learning perspective</article-title><source>Bioinformatics</source><year>2014</year><volume>30</volume><issue>17</issue><fpage>609</fpage><lpage>16</lpage><pub-id pub-id-type="doi">10.1093/bioinformatics/btu472</pub-id></element-citation></ref><ref id="CR8"><label>8</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Robinson</surname><given-names>PN</given-names></name><name><surname>Mundlos</surname><given-names>S</given-names></name></person-group><article-title>The human phenotype ontology</article-title><source>Clin Genet</source><year>2010</year><volume>77</volume><issue>6</issue><fpage>525</fpage><lpage>34</lpage><pub-id pub-id-type="doi">10.1111/j.1399-0004.2010.01436.x</pub-id><pub-id pub-id-type="pmid">20412080</pub-id></element-citation></ref><ref id="CR9"><label>9</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Moreau</surname><given-names>Y</given-names></name><name><surname>Tranchevent</surname><given-names>LC</given-names></name></person-group><article-title>Computational tools for prioritizing candidate genes: boosting disease gene discovery</article-title><source>Nat Rev Genet</source><year>2012</year><volume>13</volume><issue>8</issue><fpage>523</fpage><lpage>36</lpage><pub-id pub-id-type="doi">10.1038/nrg3253</pub-id><pub-id pub-id-type="pmid">22751426</pub-id></element-citation></ref><ref id="CR10"><label>10</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Clark</surname><given-names>WT</given-names></name><name><surname>Radivojac</surname><given-names>P</given-names></name></person-group><article-title>Information-theoretic evaluation of predicted ontological annotations</article-title><source>Bioinformatics</source><year>2013</year><volume>29</volume><issue>13</issue><fpage>53</fpage><lpage>61</lpage><pub-id pub-id-type="doi">10.1093/bioinformatics/btt228</pub-id></element-citation></ref><ref id="CR11"><label>11</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Bairoch</surname><given-names>A</given-names></name><name><surname>Apweiler</surname><given-names>R</given-names></name><name><surname>Wu</surname><given-names>CH</given-names></name><name><surname>Barker</surname><given-names>WC</given-names></name><name><surname>Boeckmann</surname><given-names>B</given-names></name><name><surname>Ferro</surname><given-names>S</given-names></name><etal></etal></person-group><article-title>The Universal Protein Resource (UniProt)</article-title><source>Nucleic Acids Res</source><year>2005</year><volume>33</volume><issue>Database issue</issue><fpage>154</fpage><lpage>9</lpage><pub-id pub-id-type="doi">10.1093/nar/gki070</pub-id></element-citation></ref><ref id="CR12"><label>12</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Huntley</surname><given-names>RP</given-names></name><name><surname>Sawford</surname><given-names>T</given-names></name><name><surname>Mutowo-Meullenet</surname><given-names>P</given-names></name><name><surname>Shypitsyna</surname><given-names>A</given-names></name><name><surname>Bonilla</surname><given-names>C</given-names></name><name><surname>Martin</surname><given-names>MJ</given-names></name><etal></etal></person-group><article-title>The GOA database: gene ontology annotation updates for 2015</article-title><source>Nucleic Acids Res</source><year>2015</year><volume>43</volume><issue>Database issue</issue><fpage>1057</fpage><lpage>63</lpage><pub-id pub-id-type="doi">10.1093/nar/gku1113</pub-id></element-citation></ref><ref id="CR13"><label>13</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Clark</surname><given-names>WT</given-names></name><name><surname>Radivojac</surname><given-names>P</given-names></name></person-group><article-title>Analysis of protein function and its prediction from amino acid sequence</article-title><source>Proteins</source><year>2011</year><volume>79</volume><issue>7</issue><fpage>2086</fpage><lpage>96</lpage><pub-id pub-id-type="doi">10.1002/prot.23029</pub-id><pub-id pub-id-type="pmid">21671271</pub-id></element-citation></ref><ref id="CR14"><label>14</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Altschul</surname><given-names>SF</given-names></name><name><surname>Madden</surname><given-names>TL</given-names></name><name><surname>Schaffer</surname><given-names>AA</given-names></name><name><surname>Zhang</surname><given-names>J</given-names></name><name><surname>Zhang</surname><given-names>Z</given-names></name><name><surname>Miller</surname><given-names>W</given-names></name><etal></etal></person-group><article-title>Gapped BLAST and PSI-BLAST: a new generation of protein database search programs</article-title><source>Nucleic Acids Res</source><year>1997</year><volume>25</volume><issue>17</issue><fpage>3389</fpage><lpage>402</lpage><pub-id pub-id-type="doi">10.1093/nar/25.17.3389</pub-id><pub-id pub-id-type="pmid">9254694</pub-id></element-citation></ref><ref id="CR15"><label>15</label><element-citation publication-type="book"><person-group person-group-type="author"><name><surname>Efron</surname><given-names>B</given-names></name><name><surname>Tibshirani</surname><given-names>RJ</given-names></name></person-group><source>An introduction to the bootstrap</source><year>1993</year><publisher-loc>New York</publisher-loc><publisher-name>Chapman & Hall</publisher-name></element-citation></ref><ref id="CR16"><label>16</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Cal</surname><given-names>S</given-names></name><name><surname>Argüelles</surname><given-names>JM</given-names></name><name><surname>Fernández</surname><given-names>PL</given-names></name><name><surname>López-Otın</surname><given-names>C</given-names></name></person-group><article-title>Identification, characterization, and intracellular processing of ADAM-TS12, a novel human disintegrin with a complex structural organization involving multiple thrombospondin-1 repeats</article-title><source>J Biol Chem</source><year>2001</year><volume>276</volume><issue>21</issue><fpage>17932</fpage><lpage>40</lpage><pub-id pub-id-type="doi">10.1074/jbc.M100534200</pub-id><pub-id pub-id-type="pmid">11279086</pub-id></element-citation></ref><ref id="CR17"><label>17</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Wolfsberg</surname><given-names>TG</given-names></name><name><surname>Straight</surname><given-names>PD</given-names></name><name><surname>Gerena</surname><given-names>RL</given-names></name><name><surname>Huovila</surname><given-names>A-PJ</given-names></name><name><surname>Primakoff</surname><given-names>P</given-names></name><name><surname>Myles</surname><given-names>DG</given-names></name><etal></etal></person-group><article-title>ADAM, a widely distributed and developmentally regulated gene family encoding membrane proteins with a disintegrin and metalloprotease domain</article-title><source>Dev Biol</source><year>1995</year><volume>169</volume><issue>1</issue><fpage>378</fpage><lpage>83</lpage><pub-id pub-id-type="doi">10.1006/dbio.1995.1152</pub-id><pub-id pub-id-type="pmid">7750654</pub-id></element-citation></ref><ref id="CR18"><label>18</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Brocker</surname><given-names>CN</given-names></name><name><surname>Vasiliou</surname><given-names>V</given-names></name><name><surname>Nebert</surname><given-names>DW</given-names></name></person-group><article-title>Evolutionary divergence and functions of the ADAM and ADAMTS gene families</article-title><source>Hum Genomics</source><year>2009</year><volume>4</volume><issue>1</issue><fpage>43</fpage><lpage>55</lpage><pub-id pub-id-type="doi">10.1186/1479-7364-4-1-43</pub-id><pub-id pub-id-type="pmid">19951893</pub-id></element-citation></ref><ref id="CR19"><label>19</label><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Wass</surname><given-names>MN</given-names></name><name><surname>Mooney</surname><given-names>SD</given-names></name><name><surname>Linial</surname><given-names>M</given-names></name><name><surname>Radivojac</surname><given-names>P</given-names></name><name><surname>Friedberg</surname><given-names>I</given-names></name></person-group><article-title>The automated function prediction SIG looks back at 2013 and prepares for 2014</article-title><source>Bioinformatics</source><year>2014</year><volume>30</volume><issue>14</issue><fpage>2091</fpage><lpage>2</lpage><pub-id pub-id-type="doi">10.1093/bioinformatics/btu117</pub-id><pub-id pub-id-type="pmid">24590444</pub-id></element-citation></ref></ref-list></back></pmc></record>Pour manipuler ce document sous Unix (Dilib)
EXPLOR_STEP=$WICRI_ROOT/Wicri/Asie/explor/AustralieFrV1/Data/Pmc/Corpus
HfdSelect -h $EXPLOR_STEP/biblio.hfd -nk 000D08 | SxmlIndent | more
Ou
HfdSelect -h $EXPLOR_AREA/Data/Pmc/Corpus/biblio.hfd -nk 000D08 | SxmlIndent | more
Pour mettre un lien sur cette page dans le réseau Wicri
{{Explor lien
|wiki= Wicri/Asie
|area= AustralieFrV1
|flux= Pmc
|étape= Corpus
|type= RBID
|clé=
|texte=
}}
| This area was generated with Dilib version V0.6.33. |  |