diff --git a/_quarto.yml b/_quarto.yml index 692de5c..8156f2c 100644 --- a/_quarto.yml +++ b/_quarto.yml @@ -86,6 +86,7 @@ website: - sections/2_academic_impact/quality.qmd - sections/2_academic_impact/use_of_code_in_research.qmd - sections/2_academic_impact/use_of_data_in_research.qmd + - sections/2_academic_impact/thematic_persistence.qmd - title: Societal Impact contents: diff --git a/references.bib b/references.bib index bb0d78c..d310541 100644 --- a/references.bib +++ b/references.bib @@ -512,6 +512,16 @@ @article{civan2010 doi = {10.1002/hec.1494} } +@article{Cobo2011AnAF, + title = {An approach for detecting, quantifying, and visualizing the evolution of a research field: A practical application to the Fuzzy Sets Theory field}, + author = {M.J. Cobo and Antonio Gabriel L{\'o}pez-Herrera and Enrique Herrera-Viedma and Francisco Herrera}, + journal = {J. Informetrics}, + year = {2011}, + volume = {5}, + pages = {146-166}, + url = {https://api.semanticscholar.org/CorpusID:9814348} +} + @misc{codeof, title = {Code of Practice}, url = {https://www.countermetrics.org/code-of-practice/}, @@ -1113,6 +1123,7 @@ @inproceedings{gialitsis2022 langid = {en} } + @inproceedings{gialitsis2022a, title = {WWW '22: The ACM Web Conference 2022}, author = {Gialitsis, Nikolaos and Kotitsas, Sotiris and Papageorgiou, Haris}, @@ -1142,7 +1153,6 @@ @inproceedings{gialitsis2022b langid = {en} } - @article{giovani2017, author = {Giovani, B.}, title = {Open Data for Research and Strategic Monitoring in the Pharmaceutical and Biotech Industry}, @@ -1153,6 +1163,7 @@ @article{giovani2017 doi = {10.5334/dsj-2017-018} } + @article{glaser_governing_2016, title = {Governing {Science}: {How} {Science} {Policy} {Shapes} {Research} {Content}}, volume = {57}, @@ -1170,7 +1181,6 @@ @article{glaser_governing_2016 pages = {117--168} } - @article{goben2020, title = {Open data repositories: Current risks and opportunities | Goben | College & Research Libraries News}, author = {Goben, Abigail and Sandusky, Robert J.}, @@ -1240,6 +1250,7 @@ @article{gordon2021 langid = {en} } + @article{gormally2012, title = {Developing a Test of Scientific Literacy Skills (TOSLS): Measuring Undergraduates{\textquoteright} Evaluation of Scientific Information and Arguments}, author = {Gormally, Cara and Brickman, Peggy and Lutz, Mary}, @@ -1263,7 +1274,6 @@ @misc{goyal_causal_2024 year = {2024} } - @article{grimme, title = {The State of Open Monographs}, author = {Grimme, Sara and Holland, Cathy and Potter, Peter and Taylor, Mike and Watkinson, Charles}, @@ -1599,6 +1609,7 @@ @article{keller2014 langid = {en} } + @article{khan2022, title = {Open science failed to penetrate academic hiring practices: a cross-sectional study}, author = {Khan, Hassan and Almoli, Elham and Franco, Marina Christ and Moher, David}, @@ -1613,7 +1624,6 @@ @article{khan2022 langid = {en} } - @article{klebel_academic_2025, title = {The academic impact of {Open} {Science}: a scoping review}, volume = {12}, @@ -2081,6 +2091,7 @@ @book{monitori2021 langid = {eng} } + @article{munafò2017, title = {A manifesto for reproducible science}, author = {{Munafò}, Marcus R. and Nosek, Brian A. and Bishop, Dorothy V. M. and Button, Katherine S. and Chambers, Christopher D. and Percie Du Sert, Nathalie and Simonsohn, Uri and Wagenmakers, Eric-Jan and Ware, Jennifer J. and Ioannidis, John P. A.}, @@ -2096,7 +2107,6 @@ @article{munafò2017 langid = {en} } - @article{munafò2018, title = {Robust research needs many lines of evidence}, author = {{Munafò}, Marcus R. and Smith, George Davey}, @@ -2142,6 +2152,7 @@ @article{nielsen2023 doi = {10.1038/s41598-023-33102-5} } + @inbook{norris2014, title = {Conceptions of Scientific Literacy: Identifying and Evaluating Their Programmatic Elements}, author = {Norris, Stephen P. and Phillips, Linda M. and Burns, David P.}, @@ -2172,7 +2183,6 @@ @article{nosek2015 note = {Publisher: American Association for the Advancement of Science} } - @article{nosek2022, title = {Replicability, Robustness, and Reproducibility in Psychological Science}, author = {Nosek, Brian A. and Hardwicke, Tom E. and Moshontz, Hannah and Allard, {Aurélien} and Corker, Katherine S. and Dreber, Anna and Fidler, Fiona and Hilgard, Joe and Kline Struhl, Melissa and Nuijten, {Michèle B.} and Rohrer, Julia M. and Romero, Felipe and Scheel, Anne M. and Scherer, Laura D. and {Schönbrodt}, Felix D. and Vazire, Simine}, @@ -2611,6 +2621,7 @@ @inbook{roberts2013 url = {https://api.taylorfrancis.com/content/chapters/edit/download?identifierName=doi&identifierValue=10.4324/9780203824696-29&type=chapterpdf} } + @inbook{roberts2013a, title = {Scientific literacy/science literacy}, author = {Roberts, Douglas A.}, @@ -2711,6 +2722,7 @@ @article{ross-hellauer2022 } + @techreport{ruiter2023, title = {Automatically Finding and Categorizing Replication Studies}, author = {Ruiter, Bob de}, @@ -2722,7 +2734,6 @@ @techreport{ruiter2023 } - @article{schmidt2009, title = {Shall we Really do it Again? The Powerful Concept of Replication is Neglected in the Social Sciences}, author = {Schmidt, Stefan}, @@ -2739,6 +2750,7 @@ @article{schmidt2009 } + @article{schnog2021, author = {Schnog, J.-J. B. and Samson, M. J. and Gans, R. O. B. and Duits, A. J.}, title = {An urgent call to raise the bar in oncology}, @@ -2751,7 +2763,6 @@ @article{schnog2021 } - @article{schoenmakers2010, title = {The technological origins of radical inventions}, author = {Schoenmakers, Wilfred and Duysters, Geert}, @@ -2765,7 +2776,6 @@ @article{schoenmakers2010 note = {Publisher: Elsevier} } - @article{schulz, title = {A network-based citation indicator of scientific performance}, author = {Schulz, Christian and Uzzi, Brian and Helbing, Dirk and Woolley-Meza, Olivia}, @@ -2773,6 +2783,7 @@ @article{schulz doi = {10.48550/arXiv.1807.04712} } + @article{shirk2012, title = {Public Participation in Scientific Research: a Framework for Deliberate Design}, author = {Shirk, Jennifer L. and Ballard, Heidi L. and Wilderman, Candie C. and Phillips, Tina and Wiggins, Andrea and Jordan, Rebecca and McCallie, Ellen and Minarchek, Matthew and Lewenstein, Bruce V. and Krasny, Marianne E. and Bonney, Rick}, @@ -2785,7 +2796,6 @@ @article{shirk2012 note = {Publisher: Resilience Alliance Inc.} } - @book{smaldino2023, title = {Modeling social behavior: mathematical and agent-based models of social dynamics and cultural evolution}, author = {Smaldino, Paul E.}, @@ -2795,6 +2805,7 @@ @book{smaldino2023 address = {Princeton} } + @techreport{soyer_what_2021, title = {What is societal impact of research? {A} literature review}, shorttitle = {What is societal impact of research?}, @@ -2807,7 +2818,6 @@ @techreport{soyer_what_2021 file = {PDF:/home/vtraag/Zotero/storage/ACU7XAP5/Literature-Review_Societal-Research-Impact.pdf:application/pdf} } - @techreport{sparceurope2019, title = {Using open and FAIR data to increase research efficiency}, author = {SPARC Europe}, @@ -2871,6 +2881,8 @@ @article{sugimoto2011 url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/asi.21568} } + + @techreport{sveinsdottir2021, title = {An Analysis of Open Science Policies in Europe, v7}, author = {Sveinsdottir, Thordis and Davidson, Joy and Proudman, Vanessa}, @@ -2882,7 +2894,6 @@ @techreport{sveinsdottir2021 } - @article{szomszor2022, title = {Overton: A bibliometric database of policy document citations}, author = {Szomszor, Martin and Adie, Euan}, @@ -2898,6 +2909,7 @@ @article{szomszor2022 } + @book{tashakkori2021, title = {Foundations of mixed methods research: integrating quantitative and qualitative approaches in the social and behavioral sciences}, author = {Tashakkori, Abbas and Johnson, R. Burke and Teddlie, Charles}, @@ -2909,7 +2921,6 @@ @book{tashakkori2021 } - @article{tattersall2018, title = {What Can Altmetric.com Tell Us About Policy Citations of Research? An Analysis of Altmetric.com Data for Research Articles from the University of Sheffield}, author = {Tattersall, Andy and Carroll, Christopher}, @@ -2924,7 +2935,6 @@ @article{tattersall2018 langid = {English} } - @article{tennant2016, title = {The academic, economic and societal impacts of Open Access: an evidence-based review}, author = {Tennant, Jonathan P. and Waldner, {François} and Jacques, Damien C. and Masuzzo, Paola and Collister, Lauren B. and Hartgerink, Chris H. J.}, @@ -2935,6 +2945,7 @@ @article{tennant2016 url = {https://f1000research.com/articles/5-632} } + @article{tiokhin_shifting_2023, title = {Shifting the {Level} of {Selection} in {Science}}, issn = {1745-6916}, @@ -2949,7 +2960,6 @@ @article{tiokhin_shifting_2023 pages = {17456916231182568} } - @article{tomkins_reviewer_2017, title = {Reviewer bias in single- versus double-blind peer review}, volume = {114}, @@ -3123,6 +3133,16 @@ @article{waltman2011 note = {PMID: 21654898} } +@article{Waltman2012ANM, + title = {A new methodology for constructing a publication-level classification system of science}, + author = {Ludo Waltman and Nees Jan van Eck}, + journal = {J. Assoc. Inf. Sci. Technol.}, + year = {2012}, + volume = {63}, + pages = {2378-2392}, + url = {https://api.semanticscholar.org/CorpusID:15589099} +} + @article{waltman2013, title = {On the calculation of percentile-based bibliometric indicators}, author = {Waltman, Ludo and Schreiber, Michael}, @@ -3322,6 +3342,16 @@ @article{wuchty2007 langid = {en} } +@article{Yan2014ResearchDM, + title = {Research dynamics: Measuring the continuity and popularity of research topics}, + author = {Erjia Yan}, + journal = {J. Informetrics}, + year = {2014}, + volume = {8}, + pages = {98-110}, + url = {https://api.semanticscholar.org/CorpusID:35965754} +} + @article{yarkoni2019, title = {The Generalizability Crisis}, author = {Yarkoni, Tal}, diff --git a/sections/2_academic_impact/thematic_persistence.qmd b/sections/2_academic_impact/thematic_persistence.qmd new file mode 100644 index 0000000..0377cfe --- /dev/null +++ b/sections/2_academic_impact/thematic_persistence.qmd @@ -0,0 +1,139 @@ +--- +title: Thematic Persistence +author: + - name: P. Stavropoulos + orcid: 0000-0003-1664-6554 + affiliations: + - ref: arc +affiliations: + - id: arc + name: Athena Research Center + city: Athena + country: Greece +--- + +::: {.callout collapse="true"} +# History + +| Version | Revision date | Revision | Author | +|---------|---------------|-------------|---------------------| +| 1.1 | 2025-08-25 | Additions | Petros Stavropoulos | +| 1.0 | 2025-05-09 | First Draft | Petros Stavropoulos | +::: + +# Description + +*Thematic Persistence* captures the ability of a research topic to remain present and influential in the scientific record over extended periods of time. It reflects the **continuity, longevity, and stability** of themes, distinguishing enduring areas of inquiry from those that are short-lived or sporadic. + +Persistent topics often indicate fields with strong conceptual foundations, long-term societal or technological relevance, or strategic importance for research policy. Conversely, non-persistent topics may point to fleeting interests or speculative research directions. + +Measuring thematic persistence helps assess the structural evolution of science, identify durable agendas, and guide funding, policy, and curriculum decisions. + +## Thematic Persistence Score (TPS) + +One way to operationalize persistence is through composite indicators such as the *Thematic Persistence Score (TPS)*. TPS combines multiple aspects of a topic’s evolution (continuity across years, growth, impact, and recency) into a single measure. + +Other methodologies apply different approaches, such as linking clusters across time periods, defining continuity typologies, or evaluating the survival of citation-based topics. + +### Measurement + +Thematic persistence can be measured through a combination of: + +- **Temporal continuity**: duration and uninterrupted presence of topics across consecutive years or periods. +- **Growth dynamics**: how the volume of publications on a topic changes over time. +- **Impact measures**: the influence of topic publications relative to their fields. +- **Structural stability**: whether a topic maintains coherence in its conceptual or citation network. +- **Recency**: whether a topic remains active in the most recent period. + +The precise operationalization depends on the chosen methodology, as outlined below. + +#### Datasources + +##### OpenAIRE Research Graph + +The [OpenAIRE Research Graph](https://graph.openaire.eu/) offers extensive metadata on publications, including: + +- **Publication year**, which is crucial for identifying consecutive topic appearances. +- **Citation metadata**, enabling FWCI computation via connected sources. + +##### Semantic Scholar + +[Semantic Scholar](https://www.semanticscholar.org/) offers full-text access and machine-readable metadata, including: + +- Year of publication +- Citation counts + +#### Existing Methodologies + +##### SciNoBo Toolkit + +The [SciNoBo Toolkit](https://scinobo.ilsp.gr/toolkit) provides essential functionalities for TPS: + +- **Field of Science (FoS) classification**: Uses a hierarchical taxonomy (6 levels) to assign topics to publications, capturing both broad disciplines and fine-grained emerging themes. This allows robust tracking of how topics evolve across scientific fields. +- **Citation Analysis**: Aggregates citation metrics across publications, facilitating computation of Field-Weighted Citation Impact (FWCI) for each topic-year combination. + +These tools make it feasible to apply the TPS metric across large bibliographic datasets with a rich contextual understanding of scientific domains. + +**TPS Formula:** + +For each topic, sequences of consecutive years are identified where the topic appears in publications. For each such sequence `s`, the score is computed as: + +$$ +\text{Score}_s = (\text{Length}_s)^{1.5} \times \text{Count}_s \times \text{Growth}_s \times \text{FWCI}_s \times \text{Recency}_s +$$ + +Where: + +- $\text{Length}_s$: Length of the sequence (in years) +- $\text{Count}_s$: Number of publications in the sequence - +- $\text{Growth}_s = \frac{\text{LastYearCount}}{\text{FirstYearCount}}$ (capped at 3) +- $\text{FWCI}_s$: Mean Field-Weighted Citation Impact for publications in the sequence +- $\text{Recency}_s = 1 + \frac{w (\text{LastYear}_s - \text{MaxYear} + 10)}{10}$, with $w$ as a recency weight (e.g. 0.2) + +The final TPS is the sum of the scores for all sequences of the topic: + +$$ +\text{TPS}_{\text{topic}} = \sum_s \text{Score}_s +$$ + +This approach emphasizes **continuity**, while integrating **growth, impact, and recency**. + +##### Longitudinal Co-word Analysis (SciMAT) + +The SciMAT framework [@Cobo2011AnAF] measures persistence by detecting **continuing themes** across consecutive time periods. + + +- Topics identified via keyword co-occurrence networks. +- Continuity measured using the **Inclusion Index**: + +$$ +\text{Inclusion}(U,V) = \frac{|U \cap V|}{\min(|U|,|V|)} +$$ + +- Topics linked across periods with high inclusion are *continuing*; absence of links indicates *new* or *discontinued* themes. + +This approach emphasizes **structural continuity of thematic vocabularies**. + +##### Direct-Citation Topic Survival + +The CWTS publication-level classification system [@Waltman2012ANM] enables persistence analysis based on citation-linked topic clusters. + +- Topics are defined via **direct citation clustering**.\ +- Persistence is measured through indicators such as: + - **Survival length**: number of years a cluster remains active.\ + - **Activity stability**: whether publication volume is maintained or growing. + +This approach measures persistence at the **topic-cluster level**, grounded in citation networks. + +##### Continuity Typologies + +The continuity framework [@Yan2014ResearchDM] quantifies persistence by categorizing topics into distinct evolutionary types: + +- **Steady**: stable over time\ +- **Concentrating**: narrowing focus while persisting\ +- **Diluting**: broadening and dispersing\ +- **Sporadic**: intermittent appearance\ +- **Emerging**: new and growing + +Continuity is evaluated by the **strength of inter-year linkages** among topic clusters.\ +This allows distinguishing different **modes of persistence** and topic evolution.