<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">ResProt</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR Res Protoc</journal-id>
      <journal-title>JMIR Research Protocols</journal-title>
      <issn pub-type="epub">1929-0748</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v10i2e13731</article-id>
      <article-id pub-id-type="pmid">33599620</article-id>
      <article-id pub-id-type="doi">10.2196/13731</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Protocol</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Protocol</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Perspectives Toward Seeking Treatment Among Patients With Psoriasis: Protocol for a Twitter Content Analysis</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Eysenbach</surname>
            <given-names>Gunther</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Wickramasinghe</surname>
            <given-names>Nilmini</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Wang</surname>
            <given-names>Cheng</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" corresp="yes">
          <name name-style="western">
            <surname>Reuter</surname>
            <given-names>Katja</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <xref rid="aff2" ref-type="aff">2</xref>
          <address>
            <institution>Department of Public Health and Preventive Medicine</institution>
            <institution>SUNY Upstate Medical University</institution>
            <addr-line>766 Irving Avenue</addr-line>
            <addr-line>Syracuse, NY, 13210</addr-line>
            <country>United States</country>
            <phone>1 315 464 1520</phone>
            <fax>1 315 464 1701</fax>
            <email>reuterk@upstate.edu</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-1559-9058</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Lee</surname>
            <given-names>Delphine</given-names>
          </name>
          <degrees>MD, PhD</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <xref rid="aff4" ref-type="aff">4</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-0063-5295</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Southern California Clinical and Translational Science Institute</institution>
        <institution>Keck School of Medicine</institution>
        <institution>University of Southern California</institution>
        <addr-line>Los Angeles, CA</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Department of Public Health and Preventive Medicine</institution>
        <institution>SUNY Upstate Medical University</institution>
        <addr-line>Syracuse, NY</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff3">
        <label>3</label>
        <institution>Division of Dermatology</institution>
        <institution>Department of Medicine</institution>
        <institution>Harbor-UCLA Medical Center</institution>
        <addr-line>Torrance, CA</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff4">
        <label>4</label>
        <institution>The Lundquist Institute</institution>
        <addr-line>Torrance, CA</addr-line>
        <country>United States</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Katja Reuter <email>reuterk@upstate.edu</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <month>2</month>
        <year>2021</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>18</day>
        <month>2</month>
        <year>2021</year>
      </pub-date>
      <volume>10</volume>
      <issue>2</issue>
      <elocation-id>e13731</elocation-id>
      <history>
        <date date-type="received">
          <day>11</day>
          <month>7</month>
          <year>2019</year>
        </date>
        <date date-type="rev-request">
          <day>9</day>
          <month>12</month>
          <year>2019</year>
        </date>
        <date date-type="rev-recd">
          <day>28</day>
          <month>2</month>
          <year>2020</year>
        </date>
        <date date-type="accepted">
          <day>5</day>
          <month>3</month>
          <year>2020</year>
        </date>
      </history>
      <copyright-statement>©Katja Reuter, Delphine Lee. Originally published in JMIR Research Protocols (http://www.researchprotocols.org), 18.02.2021.</copyright-statement>
      <copyright-year>2021</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Research Protocols, is properly cited. The complete bibliographic information, a link to the original publication on http://www.researchprotocols.org, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="http://www.researchprotocols.org/2021/2/e13731/" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>Psoriasis is an autoimmune disease estimated to affect more than 6 million adults in the United States. It poses a significant public health problem and contributes to rising health care costs, affecting people’s quality of life and ability to work. Previous research showed that nontreatment and undertreatment of patients with psoriasis remain a significant problem. Perspectives of patients toward seeking psoriasis treatment are understudied. Social media offers a new data source of user-generated content. Researchers suggested that the social network Twitter may serve as a rich avenue for exploring how patients communicate about their health issues.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>The objective of this study is to conduct a content analysis of Twitter posts (in English) published by users in the United States between February 1, 2016, and October 31, 2018, to examine perspectives that potentially influence the treatment decision among patients with psoriasis.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>User-generated Twitter posts that include keywords related to psoriasis will be analyzed using text classifiers to identify themes related to the research questions. We will use Symplur Signals, a health care social media analytics platform, to access the Twitter data. We will use descriptive statistics to analyze the data and identify the most prevalent topics in the Twitter content among people with psoriasis.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>This study is supported by the National Center for Advancing Translational Science through a Clinical and Translational Science Award award. Study approval was obtained from the institutional review board at the University of Southern California. Data extraction and cleaning are complete. For the time period from February 1, 2016, to October 31, 2018, we obtained 95,040 Twitter posts containing terms related to “psoriasis” from users in the United States published in English. After removing duplicates, retweets, and non-English tweets, we found that 75.51% (52,301/69,264) of the psoriasis-related posts were sent by commercial or bot-like accounts, while 16,963 posts were noncommercial and will be included in the analysis to assess the patient perspective. Analysis was completed in Summer 2020.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>This protocol paper provides a detailed description of a social media research project including the process of data extraction, cleaning, and analysis. It is our goal to contribute to the development of more transparent social media research efforts. Our findings will shed light on whether Twitter provides a promising data source for garnering patient perspective data about psoriasis treatment decisions. The data will also help to determine whether Twitter might serve as a potential outreach platform for raising awareness of psoriasis and treatment options among patients and implementing related health interventions.</p>
        </sec>
        <sec sec-type="registered-report">
          <title>International Registered Report Identifier (IRRID)</title>
          <p>DERR1-10.2196/13731</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>infodemiology</kwd>
        <kwd>infoveillance</kwd>
        <kwd>internet</kwd>
        <kwd>surveillance</kwd>
        <kwd>patient opinion</kwd>
        <kwd>psoriasis, treatment</kwd>
        <kwd>Twitter</kwd>
        <kwd>social media</kwd>
        <kwd>social network</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <sec>
        <title>Background and Rationale</title>
        <p>Psoriasis is an autoimmune disease that causes patches of thick red skin and silvery scales and affects more than 6 million adults in the United States [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref2">2</xref>]. The condition can impact the quality of life and psychological and social functioning [<xref ref-type="bibr" rid="ref3">3</xref>] and poses a significant public health problem [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref5">5</xref>]. A total direct cost of $649.6 million for approximately 1.4 million individuals with clinically significant psoriasis was reported [<xref ref-type="bibr" rid="ref6">6</xref>], affecting employment and individuals’ ability to work [<xref ref-type="bibr" rid="ref7">7</xref>]. The condition occurs mostly in adults, men and women alike (ages 18 years and older) but can also affect children and is most common in people aged 50 to 69 years [<xref ref-type="bibr" rid="ref4">4</xref>].</p>
        <p>Previous research showed that nontreatment and undertreatment of patients with psoriasis remain a significant problem in the United States [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref9">9</xref>]. Despite several treatment options, 9% to 30% of patients with severe and nearly 50% of patients with mild psoriasis symptoms do not receive treatment, and patients report widespread treatment dissatisfaction [<xref ref-type="bibr" rid="ref10">10</xref>]. More specifically, up to 30% of patients with severe and nearly 50% of patients with mild psoriasis symptoms do not receive treatment [<xref ref-type="bibr" rid="ref8">8</xref>]. Known barriers to seeking treatment include a limited understanding of the disease, insurance coverage and out-of-pocket costs, and safety profile concerns [<xref ref-type="bibr" rid="ref11">11</xref>]. However, perspectives among psoriasis patients toward seeking treatment are understudied. For example, a PubMed search including the terms “psoriasis,” “patients,” and “seeking treatment” results in 4 reports, of which only one identifies treatment-seeking motivations of psoriasis patients [<xref ref-type="bibr" rid="ref12">12</xref>]. In this study, we define perspective as any expression of thought, viewpoint, or attitude toward health issues and concerns. Efforts that improve the understanding of patients’ perspectives could inform and enhance advocacy and education to ensure that effective treatments are accessible to these patients.</p>
      </sec>
      <sec>
        <title>Social Media and Health Research</title>
        <p>Social media includes widely accessible web-based and mobile technologies that allow users to participate in social networking and view, create, and share information online [<xref ref-type="bibr" rid="ref13">13</xref>]. These communication tools provide a unique source for data mining of health conditions and concerns, serving as a massive focus group of sorts [<xref ref-type="bibr" rid="ref14">14</xref>-<xref ref-type="bibr" rid="ref16">16</xref>]; 72% of American adults use at least one social media platform [<xref ref-type="bibr" rid="ref17">17</xref>].</p>
        <p>The emergence of social media has created new sources of analyzable data [<xref ref-type="bibr" rid="ref11">11</xref>] and led to new research fields (ie, infodemiology and infoveillance) [<xref ref-type="bibr" rid="ref16">16</xref>,<xref ref-type="bibr" rid="ref18">18</xref>]. The data social media users generate through their online activities are referred to as their digital footprint [<xref ref-type="bibr" rid="ref19">19</xref>] or social mediome [<xref ref-type="bibr" rid="ref20">20</xref>]. Recently, the US Food and Drug Administration (FDA) encouraged the use of unstructured patient-generated health data (PGHD) from different sources including social media to generate insight into patient-experienced outcomes in the real world [<xref ref-type="bibr" rid="ref21">21</xref>-<xref ref-type="bibr" rid="ref23">23</xref>]. On Twitter, for example, health surveillance researchers have used these data to gain insight into public perspectives on a variety of diseases and health topics such as influenza, autism, schizophrenia, smoking, HIV/AIDS, and sun-related issues and skin cancer [<xref ref-type="bibr" rid="ref24">24</xref>-<xref ref-type="bibr" rid="ref30">30</xref>]. In some cases, social media user data demonstrated a correlation between the disease prevalence and frequency with which Twitter users discussed a disease [<xref ref-type="bibr" rid="ref31">31</xref>]. The use of PGHD from social media offers a new opportunity to learn about patients’ disease experience and networks that are not otherwise easily captured through traditional surveys or administrative data [<xref ref-type="bibr" rid="ref32">32</xref>].</p>
      </sec>
      <sec>
        <title>The Social Network Twitter</title>
        <p>Nearly 22% of US adults use the social network Twitter including Hispanics (25%), Blacks (24%), and Whites (21%); more than 40% use the platform daily [<xref ref-type="bibr" rid="ref17">17</xref>]. Twitter users can post short messages (tweets) of up to 280 characters and search for any public message and further engage with these tweets (ie, they can like, reply, and retweet [share] them). Twitter is a primarily public social network; by default, basic Twitter account information such as the profile name, description, and location are public unless a user decides to opt out and make an account private. Due to the more public nature of Twitter, previous research suggested that Twitter provides a “rich and promising avenue for exploring how patients conceptualize and communicate about their specific health issues” [<xref ref-type="bibr" rid="ref33">33</xref>]. The increasing use of Twitter among members of disease communities is further evidenced by the abundance of disease and health topic hashtags used in the messages [<xref ref-type="bibr" rid="ref34">34</xref>-<xref ref-type="bibr" rid="ref36">36</xref>]. A hashtag is a word or phrase preceded by a hash or pound sign (#) and used to identify messages on a specific topic (eg, #psoriasis, #skinchat, #PsoriaticArthritis). These hashtags are used by Twitter users to assign their message to a topic and join ongoing conversations. Users can click on a hashtag and view all of the messages that include the same hashtag and, hence, discuss the same topic. This allows users to form online communities and share their health concerns, disease experience, and questions with like-minded users [<xref ref-type="bibr" rid="ref37">37</xref>]. However, there is little information about the use of social media among psoriasis patients.</p>
      </sec>
      <sec>
        <title>Previous Work</title>
        <p>Few studies have examined social media content about psoriasis. Three studies of YouTube videos showed that misinformation is prevalent on social media and patients are exposed to a wide variety of information, with most of the content being of low quality [<xref ref-type="bibr" rid="ref38">38</xref>-<xref ref-type="bibr" rid="ref40">40</xref>]. Another study of dermatology-related content including psoriasis on the photo-sharing social network Instagram demonstrated that information by private offices, cosmetic products, and some patient advocacy groups dominates the user experience, while the use of a large number of hashtags related to dermatological conditions suggests that people use Instagram to post personal experiences with skin conditions [<xref ref-type="bibr" rid="ref41">41</xref>]. While preparations for this study were underway, Menzies et al [<xref ref-type="bibr" rid="ref42">42</xref>] published their Twitter analysis of attitudes toward psoriasis treatment among Twitter users. However, there were a few issues with their methodology that weakened their findings [<xref ref-type="bibr" rid="ref43">43</xref>]. One of the major issues pertained to the fact that the authors did not account for commercial and bot-like content within their dataset. Bots (robots) are purely automated accounts or human-assisted automated accounts (cyborgs) [<xref ref-type="bibr" rid="ref44">44</xref>-<xref ref-type="bibr" rid="ref48">48</xref>]. Identifying commercial and bot-like content, which is abundant on social media [<xref ref-type="bibr" rid="ref44">44</xref>], is critical to discern patients’ perspectives. Furthermore, the authors did not discuss whether and how they controlled for bias introduced by Twitter posts from commercial groups and bots in their analysis. As part of the preparations and data collection for our study, we found that 75.51% (52,301/69,264) of psoriasis tweets in English sent between February 2016 and October 2018 by users in the United States were commercial or bot-like in nature. To our knowledge, there are no additional studies that have used Twitter to gain a more profound understanding of patients’ attitudes toward seeking psoriasis treatment.</p>
      </sec>
      <sec>
        <title>Study Objective and Research Questions</title>
        <p>The objective of this study is to conduct a content analysis of Twitter posts (in English) published by users in the United States between February 1, 2016, to October 31, 2018, to examine perspectives that potentially influence the treatment decision among patients with psoriasis. We intend to answer the following research questions:</p>
        <list list-type="order">
          <list-item>
            <p>What perspectives toward seeking treatment are being expressed by psoriasis patients on Twitter?</p>
          </list-item>
          <list-item>
            <p>What are the demographics (ie, gender, race/ethnicity) of these psoriasis patients on Twitter?</p>
          </list-item>
          <list-item>
            <p>What is the volume of unique Twitter users who talk about this topic?</p>
          </list-item>
          <list-item>
            <p>What are the predominant themes in the conversations among psoriasis patients?</p>
          </list-item>
          <list-item>
            <p>For commercial and bot-like tweets, what types of treatments are being promoted?</p>
          </list-item>
        </list>
        <p>This protocol paper provides a detailed description of a social media research project including the process of data extraction, cleaning, and analysis. It is our goal to contribute to the development of more transparent social media research efforts. Our findings will shed light on whether Twitter provides a promising data source for garnering patient perspective data about psoriasis treatment decisions. The data will also help to determine whether Twitter might serve as a potential outreach platform for raising awareness of psoriasis and treatment options among patients and for implementing related health interventions.</p>
      </sec>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Study Type</title>
        <p>This is a qualitative study that will analyze user-generated posts about psoriasis from the social network Twitter.</p>
      </sec>
      <sec>
        <title>Data Source</title>
        <p>Twitter posts in English containing terms related to psoriasis will be obtained for the time period from February 1, 2016, to October 31, 2018. To access public Twitter user data, we will use Symplur Signals [<xref ref-type="bibr" rid="ref49">49</xref>], a health care social media analytics company that maintains the largest publicly available database of health care and disease-related conversations with the globally recognized Healthcare Hashtag Project. Symplur Signals extracts data from the Twitter REST API (representational state transfer application programming interface) and makes it available to researchers; it is commonly used in peer-reviewed research [<xref ref-type="bibr" rid="ref50">50</xref>-<xref ref-type="bibr" rid="ref54">54</xref>]. Symplur Signals data are updated daily and easily sortable by social media user type (eg, patient, physician, health care organization), location and time zone, language, disease/health interests, and Twitter message content. The location of the users (limited to users within the United States) will be determined using a mapped location filter as defined by Gnip Inc, a social media data provider, and based on the Profile Geo 2.0 algorithm [<xref ref-type="bibr" rid="ref55">55</xref>]. That algorithm uses a number of data points to determine a user’s location including the self-reported bio location in the Twitter user profile and geotracking data if available. We extracted data from Twitter through the Symplur Signals user interface, searching for the keyword and hashtags listed in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>. The data were provided in an Excel (Microsoft Corp) file, which we further analyzed on local university computers.</p>
      </sec>
      <sec>
        <title>Search Filters</title>
        <p>We will use a framework for data collection, quality assessment, and reporting standards as well as for developing search filters for social media data as previously suggested by Kim et al [<xref ref-type="bibr" rid="ref56">56</xref>]. The root terms we will use to collect the sample of tweets are listed in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>. These terms can appear in the post or in an accompanying hashtag, for example, “psoriasis” or #PsoriasisChat. We will select keywords and hashtags based on expert knowledge (clinicians, social media experts) and use a systematic search of topic-related language based on data in Symplur Signals.</p>
      </sec>
      <sec>
        <title>Data Cleaning and Debiasing</title>
        <p>The following types of irrelevant tweets will be excluded: retweets (ie, messages shared by Twitter users that other users composed) and non-English language tweets identified using the Liu method. Liu et al [<xref ref-type="bibr" rid="ref57">57</xref>] developed and evaluated a web-based language identification tool called langid.py that uses natural language processing techniques and assists with text categorization in specific languages. They showed that the tool maintains consistently high accuracy. Furthermore, we will use the program Botometer (formerly BotOrNot) to identify Twitter accounts by social bots or commercial groups that could possibly influence the results and introduce bias [<xref ref-type="bibr" rid="ref58">58</xref>,<xref ref-type="bibr" rid="ref59">59</xref>]. Automated accounts on Twitter created by industry groups and private companies promote specific ideas or products and, thus, influence discussions. Botometer is a publicly available service launched in 2014 and includes more than 1000 variables to assess the extent to which a Twitter account exhibits characteristics of social bots [<xref ref-type="bibr" rid="ref60">60</xref>]. Variables include the account network (ie, diffusion patterns), user data (ie, metadata), friends (ie, account’s contacts), tweet rate, and sentiment and content of the account messages. The classification system generates a score that determines the likelihood of any one account being a social bot. Davis et al [<xref ref-type="bibr" rid="ref60">60</xref>] demonstrated that the program scores a detection accuracy above 95%. If an account is identified as a social bot, that account and any tweets produced from that account will be removed from our dataset so we can focus on analyzing patient’s perspectives.</p>
      </sec>
      <sec>
        <title>Data Privacy and Confidentiality</title>
        <p>All analyses will adhere to the terms and conditions, terms of use, and privacy policies of Twitter. We will further abide by University of Southern California (USC) institutional review board (IRB) regulations and the USC Privacy of Personal Information policy.</p>
        <p>All data will be entered into a computer and database that is password protected. The study data will be collected using the system Research Electronic Data Capture at USC, which is a secure, web-based app designed to support data capture for research studies. Provision of data to the IRB, National Institutes of Health (NIH), and FDA is facilitated by this database system.</p>
        <p>Any identifying and personal health information will be redacted from the dataset by the coders. Information that might identify a contributor’s identity will be redacted from any report developed to share the findings, and any Twitter posts we include in publications will be paraphrased to protect the privacy of the users.</p>
      </sec>
      <sec>
        <title>Data Analysis</title>
        <p>We will use a standard coding approach for characterizing the Twitter messages and users. Two independent team members will be responsible for coding based on a set of a priori classifiers listed in <xref ref-type="supplementary-material" rid="app2">Multimedia Appendix 2</xref> and <xref ref-type="supplementary-material" rid="app3">3</xref>. Information available in a user’s Twitter profile (ie, username, description, avatar image) will be used to characterize the user of the Twitter account who generated the post to determine if the individual is a psoriasis patient (<xref ref-type="supplementary-material" rid="app3">Multimedia Appendix 3</xref>). In other words, we will characterize a Twitter user as a psoriasis patient if they specifically mention being a patient in their description or previous tweets. We will further code the person’s gender and race/ethnicity (White person versus person of color) if the Twitter profile contains sufficient information to do so.</p>
        <p>We will then code the Twitter messages from psoriasis patients (<xref ref-type="supplementary-material" rid="app2">Multimedia Appendix 2</xref>). Individual Twitter posts will be classified as posts originating from these patients either if the user who authored the message was already classified as a psoriasis patient through examination of their Twitter profile or if the post mentioned psoriasis in the first person (eg, “Haven’t felt myself lately. Asked my doc about an alternative treatment plan today.”) We will analyze the messages from these patients to identify the health issues and concerns they express (<xref ref-type="supplementary-material" rid="app2">Multimedia Appendix 2</xref>).</p>
        <p>Cohen kappa will be calculated for each code category to assess interrater reliability [<xref ref-type="bibr" rid="ref61">61</xref>,<xref ref-type="bibr" rid="ref62">62</xref>]. Once we establish concordance in the coder’s classification with Cohen kappa greater than .80 for each coding category, the remaining data will be divided between the two coders. The project principal investigators will help to establish consensus in instances where coders disagree.</p>
      </sec>
      <sec>
        <title>Statistical Analysis</title>
        <p>This study will rely on public, anonymized data and adhere to the terms and conditions, terms of use, and privacy policies of Twitter. The proposed work received IRB approval from the authors’ university.</p>
        <p>We will use descriptive statistics to analyze the data and identify the most prevalent topics in the Twitter content. Units of analysis will be unique terms in posts as well as the number of Twitter messages and users (ie, patients). We will describe the patient characteristics focusing on gender and race/ethnicity, as displayed on Twitter. For each tweet theme analysis, we will present findings in a confusion matrix where the diagonal line indicates the prevalence of a topic and the off-diagonal lines indicate topic overlap. The number of posts containing 2 or more topics would be found at the intersection of the matrix for these topics. Representative examples of tweets within each category will be selected to illustrate additional themes and will be shown as paraphrased quotes to protect users’ privacy.</p>
      </sec>
      <sec>
        <title>Risk Analysis</title>
        <p>The described work presents minimal risk research. We will use public user data from the social network Twitter. Patient identifiers do not apply. Identifiable information such as human subjects’ names and Twitter handles will not be included in the analysis dataset.</p>
      </sec>
      <sec>
        <title>Dissemination of Study Findings</title>
        <p>The authors plan to publish the study findings in a peer-reviewed journal and at topic-related conferences (to be determined at a later date). All listed authors and/or contributors are compliant with guidelines outlined by the International Committee of Medical Journal Editors for author inclusion in a published work.</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <p>Study approval was obtained from the IRB at USC (protocol HS-18-00867). Data extraction and cleaning are complete. For the time period from February 1, 2016, to October 31, 2018, we obtained 95,040 Twitter posts containing terms related to psoriasis from users in the United States published in English. After removing duplicates, retweets, and non-English tweets, we found that 75.51% (52,301/69,264) of the psoriasis-related posts were sent by commercial or bot-like accounts, while 16,963 posts were noncommercial and will be included in the analysis to determine the patient perspective (see <xref ref-type="supplementary-material" rid="app4">Multimedia Appendix 4</xref> for detailed data extraction and cleaning flow diagram). Analysis was completed in Summer 2020.</p>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Limitations</title>
        <p>This exploratory pilot study is limited to Twitter conversations from people who use words and hashtags related to psoriasis in their Twitter posts. As a result, we will only include those patients’ posts in the dataset who are familiar with the term “psoriasis” and not include posts from patients who might talk about their disease experience on Twitter but don’t include any of these words.</p>
        <p>The generalizability of the study is somewhat limited, because Twitter messages from locations outside of the United States and messages in other, non-English languages will not be included. We also recognize that this type of social media research favors those with internet access and could, therefore, lead to potential bias in the research data. Twitter users tend to be younger (38% are aged 18 to 29 years), college graduates (32%), and located in urban areas (26%) [<xref ref-type="bibr" rid="ref17">17</xref>].</p>
      </sec>
      <sec>
        <title>Practical Significance</title>
        <p>If successful, our findings will shed light on whether Twitter provides a promising data source for garnering patients’ perspectives about psoriasis treatment decisions. The data will also help to determine whether Twitter might serve as a potential outreach platform for raising awareness of psoriasis and treatment options among patients and implementing related health interventions. This protocol paper provides a detailed description of a social media research project including the process of data extraction, cleaning, and analysis. It is our goal to contribute to the development of more transparent social media research efforts.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group>
      <supplementary-material id="app1">
        <label>Multimedia Appendix 1</label>
        <p>Keywords and hashtags to assess attitudes toward treatment among patients with psoriasis on Twitter. The selection is based on data from Symplur Signals.</p>
        <media xlink:href="resprot_v10i2e13731_app1.pdf" xlink:title="PDF File  (Adobe PDF File), 83 KB"/>
      </supplementary-material>
      <supplementary-material id="app2">
        <label>Multimedia Appendix 2</label>
        <p>Code categories to identify main themes in Twitter posts related to psoriasis.</p>
        <media xlink:href="resprot_v10i2e13731_app2.pdf" xlink:title="PDF File  (Adobe PDF File), 71 KB"/>
      </supplementary-material>
      <supplementary-material id="app3">
        <label>Multimedia Appendix 3</label>
        <p>Code categories to classify Twitter users.</p>
        <media xlink:href="resprot_v10i2e13731_app3.pdf" xlink:title="PDF File  (Adobe PDF File), 59 KB"/>
      </supplementary-material>
      <supplementary-material id="app4">
        <label>Multimedia Appendix 4</label>
        <p>Data extraction and cleaning flow diagram.</p>
        <media xlink:href="resprot_v10i2e13731_app4.pdf" xlink:title="PDF File  (Adobe PDF File), 53 KB"/>
      </supplementary-material>
    </app-group>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">FDA</term>
          <def>
            <p>US Food and Drug Administration</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">IRB</term>
          <def>
            <p>institutional review board</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">NIH</term>
          <def>
            <p>National Institutes of Health</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">PGHD</term>
          <def>
            <p>patient-generated health data</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb5">REST API</term>
          <def>
            <p>representational state transfer application programming interface</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb6">USC</term>
          <def>
            <p>University of Southern California</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>The development of the study protocol and implementation of the study have been supported by the Southern California Clinical and Translational Science Institute through grant UL1TR000130 from the National Center for Advancing Translational Sciences of the NIH. The content is solely the responsibility of the authors and does not necessarily represent the official views of the NIH. The authors further thank Praveen Angyan for his assistance with extracting and cleaning the data.</p>
    </ack>
    <fn-group>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="web">
          <article-title>Psoriasis</article-title>
          <source>US Centers for Disease Control and Prevention</source>
          <access-date>2019-07-08</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.cdc.gov/psoriasis/index.htm">https://www.cdc.gov/psoriasis/index.htm</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Helmick</surname>
              <given-names>CG</given-names>
            </name>
            <name name-style="western">
              <surname>Lee-Han</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Hirsch</surname>
              <given-names>SC</given-names>
            </name>
            <name name-style="western">
              <surname>Baird</surname>
              <given-names>TL</given-names>
            </name>
            <name name-style="western">
              <surname>Bartlett</surname>
              <given-names>CL</given-names>
            </name>
          </person-group>
          <article-title>Prevalence of psoriasis among adults in the U.S.: 2003-2006 and 2009-2010 National Health and Nutrition Examination Surveys</article-title>
          <source>Am J Prev Med</source>
          <year>2014</year>
          <month>07</month>
          <volume>47</volume>
          <issue>1</issue>
          <fpage>37</fpage>
          <lpage>45</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/24746373"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.amepre.2014.02.012</pub-id>
          <pub-id pub-id-type="medline">24746373</pub-id>
          <pub-id pub-id-type="pii">S0749-3797(14)00108-1</pub-id>
          <pub-id pub-id-type="pmcid">PMC4653077</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Rapp</surname>
              <given-names>SR</given-names>
            </name>
            <name name-style="western">
              <surname>Exum</surname>
              <given-names>ML</given-names>
            </name>
            <name name-style="western">
              <surname>Reboussin</surname>
              <given-names>DM</given-names>
            </name>
            <name name-style="western">
              <surname>Feldman</surname>
              <given-names>SR</given-names>
            </name>
            <name name-style="western">
              <surname>Fleischer</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Clark</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>The physical, psychological and social impact of psoriasis</article-title>
          <source>J Health Psychol</source>
          <year>1997</year>
          <month>10</month>
          <volume>2</volume>
          <issue>4</issue>
          <fpage>525</fpage>
          <lpage>537</lpage>
          <pub-id pub-id-type="doi">10.1177/135910539700200409</pub-id>
          <pub-id pub-id-type="medline">22013093</pub-id>
          <pub-id pub-id-type="pii">2/4/525</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="web">
          <article-title>Global report on psoriasis</article-title>
          <source>World Health Organization</source>
          <year>2016</year>
          <access-date>2021-01-23</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://apps.who.int/iris/bitstream/handle/10665/204417/9789241565189_eng.pdf">https://apps.who.int/iris/bitstream/handle/10665/204417/9789241565189_eng.pdf</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chandran</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Raychaudhuri</surname>
              <given-names>SP</given-names>
            </name>
          </person-group>
          <article-title>Geoepidemiology and environmental factors of psoriasis and psoriatic arthritis</article-title>
          <source>J Autoimmun</source>
          <year>2010</year>
          <month>05</month>
          <volume>34</volume>
          <issue>3</issue>
          <fpage>J314</fpage>
          <lpage>J321</lpage>
          <pub-id pub-id-type="doi">10.1016/j.jaut.2009.12.001</pub-id>
          <pub-id pub-id-type="medline">20034760</pub-id>
          <pub-id pub-id-type="pii">S0896-8411(09)00159-0</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Javitz</surname>
              <given-names>HS</given-names>
            </name>
            <name name-style="western">
              <surname>Ward</surname>
              <given-names>MM</given-names>
            </name>
            <name name-style="western">
              <surname>Farber</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Nail</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Vallow</surname>
              <given-names>SG</given-names>
            </name>
          </person-group>
          <article-title>The direct cost of care for psoriasis and psoriatic arthritis in the United States</article-title>
          <source>J Am Acad Dermatol</source>
          <year>2002</year>
          <month>06</month>
          <volume>46</volume>
          <issue>6</issue>
          <fpage>850</fpage>
          <lpage>860</lpage>
          <pub-id pub-id-type="doi">10.1067/mjd.2002.119669</pub-id>
          <pub-id pub-id-type="medline">12063481</pub-id>
          <pub-id pub-id-type="pii">S0190-9622(02)00004-X</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Fowler</surname>
              <given-names>JF</given-names>
            </name>
            <name name-style="western">
              <surname>Duh</surname>
              <given-names>MS</given-names>
            </name>
            <name name-style="western">
              <surname>Rovba</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Buteau</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Pinheiro</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Lobo</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Sung</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Doyle</surname>
              <given-names>JJ</given-names>
            </name>
            <name name-style="western">
              <surname>Swensen</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Mallett</surname>
              <given-names>DA</given-names>
            </name>
            <name name-style="western">
              <surname>Kosicki</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>The impact of psoriasis on health care costs and patient work loss</article-title>
          <source>J Am Acad Dermatol</source>
          <year>2008</year>
          <month>11</month>
          <volume>59</volume>
          <issue>5</issue>
          <fpage>772</fpage>
          <lpage>780</lpage>
          <pub-id pub-id-type="doi">10.1016/j.jaad.2008.06.043</pub-id>
          <pub-id pub-id-type="medline">19119095</pub-id>
          <pub-id pub-id-type="pii">S0190-9622(08)01014-1</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Armstrong</surname>
              <given-names>AW</given-names>
            </name>
            <name name-style="western">
              <surname>Robertson</surname>
              <given-names>AD</given-names>
            </name>
            <name name-style="western">
              <surname>Wu</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Schupp</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Lebwohl</surname>
              <given-names>MG</given-names>
            </name>
          </person-group>
          <article-title>Undertreatment, treatment trends, and treatment dissatisfaction among patients with psoriasis and psoriatic arthritis in the United States: findings from the National Psoriasis Foundation surveys, 2003-2011</article-title>
          <source>JAMA Dermatol</source>
          <year>2013</year>
          <month>10</month>
          <volume>149</volume>
          <issue>10</issue>
          <fpage>1180</fpage>
          <lpage>1185</lpage>
          <pub-id pub-id-type="doi">10.1001/jamadermatol.2013.5264</pub-id>
          <pub-id pub-id-type="medline">23945732</pub-id>
          <pub-id pub-id-type="pii">1729130</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kurd</surname>
              <given-names>SK</given-names>
            </name>
            <name name-style="western">
              <surname>Gelfand</surname>
              <given-names>JM</given-names>
            </name>
          </person-group>
          <article-title>The prevalence of previously diagnosed and undiagnosed psoriasis in US adults: results from NHANES 2003-2004</article-title>
          <source>J Am Acad Dermatol</source>
          <year>2009</year>
          <month>02</month>
          <volume>60</volume>
          <issue>2</issue>
          <fpage>218</fpage>
          <lpage>224</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/19022533"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.jaad.2008.09.022</pub-id>
          <pub-id pub-id-type="medline">19022533</pub-id>
          <pub-id pub-id-type="pii">S0190-9622(08)01201-2</pub-id>
          <pub-id pub-id-type="pmcid">PMC4762027</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Stern</surname>
              <given-names>RS</given-names>
            </name>
            <name name-style="western">
              <surname>Nijsten</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Feldman</surname>
              <given-names>SR</given-names>
            </name>
            <name name-style="western">
              <surname>Margolis</surname>
              <given-names>DJ</given-names>
            </name>
            <name name-style="western">
              <surname>Rolstad</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Psoriasis is common, carries a substantial burden even when not extensive, and is associated with widespread treatment dissatisfaction</article-title>
          <source>J Investig Dermatol Symp Proc</source>
          <year>2004</year>
          <month>03</month>
          <volume>9</volume>
          <issue>2</issue>
          <fpage>136</fpage>
          <lpage>139</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S0022-202X(15)53000-5"/>
          </comment>
          <pub-id pub-id-type="doi">10.1046/j.1087-0024.2003.09102.x</pub-id>
          <pub-id pub-id-type="medline">15083780</pub-id>
          <pub-id pub-id-type="pii">S0022-202X(15)53000-5</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Vanderpuye-Orgle</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Zhao</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Lu</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Shrestha</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Sexton</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Seabury</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Lebwohl</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Evaluating the economic burden of psoriasis in the United States</article-title>
          <source>J Am Acad Dermatol</source>
          <year>2015</year>
          <month>06</month>
          <volume>72</volume>
          <issue>6</issue>
          <fpage>961</fpage>
          <lpage>967</lpage>
          <pub-id pub-id-type="doi">10.1016/j.jaad.2015.02.1099</pub-id>
          <pub-id pub-id-type="medline">25882886</pub-id>
          <pub-id pub-id-type="pii">S0190-9622(15)01338-9</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Golbari</surname>
              <given-names>NM</given-names>
            </name>
            <name name-style="western">
              <surname>Porter</surname>
              <given-names>ML</given-names>
            </name>
            <name name-style="western">
              <surname>Kimball</surname>
              <given-names>AB</given-names>
            </name>
          </person-group>
          <article-title>Discordance between patient reported motivations to seek treatment for psoriasis compared to television advertisements</article-title>
          <source>J Drugs Dermatol</source>
          <year>2018</year>
          <month>08</month>
          <day>01</day>
          <volume>17</volume>
          <issue>8</issue>
          <fpage>886</fpage>
          <lpage>887</lpage>
          <pub-id pub-id-type="medline">30124728</pub-id>
          <pub-id pub-id-type="pii">S1545961618P0886X</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Obar</surname>
              <given-names>JA</given-names>
            </name>
            <name name-style="western">
              <surname>Wildman</surname>
              <given-names>SS</given-names>
            </name>
          </person-group>
          <article-title>Social media definition and the governance challenge: an introduction to the special issue</article-title>
          <source>SSRN Journal</source>
          <year>2015</year>
          <volume>39</volume>
          <issue>9</issue>
          <fpage>745</fpage>
          <lpage>750</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://ssrn.com/abstract=2647377"/>
          </comment>
          <pub-id pub-id-type="doi">10.2139/ssrn.2647377</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sinnenberg</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Buttenheim</surname>
              <given-names>AM</given-names>
            </name>
            <name name-style="western">
              <surname>Padrez</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Mancheno</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Ungar</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Merchant</surname>
              <given-names>RM</given-names>
            </name>
          </person-group>
          <article-title>Twitter as a tool for health research: a systematic review</article-title>
          <source>Am J Public Health</source>
          <year>2017</year>
          <month>12</month>
          <volume>107</volume>
          <issue>1</issue>
          <fpage>e1</fpage>
          <lpage>e8</lpage>
          <pub-id pub-id-type="doi">10.2105/AJPH.2016.303512</pub-id>
          <pub-id pub-id-type="medline">27854532</pub-id>
          <pub-id pub-id-type="pmcid">PMC5308155</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ayers</surname>
              <given-names>JW</given-names>
            </name>
            <name name-style="western">
              <surname>Althouse</surname>
              <given-names>BM</given-names>
            </name>
            <name name-style="western">
              <surname>Dredze</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Could behavioral medicine lead the web data revolution?</article-title>
          <source>JAMA</source>
          <year>2014</year>
          <month>04</month>
          <day>9</day>
          <volume>311</volume>
          <issue>14</issue>
          <fpage>1399</fpage>
          <lpage>1400</lpage>
          <pub-id pub-id-type="doi">10.1001/jama.2014.1505</pub-id>
          <pub-id pub-id-type="medline">24577162</pub-id>
          <pub-id pub-id-type="pii">1838433</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zeraatkar</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Ahmadi</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Trends of infodemiology studies: a scoping review</article-title>
          <source>Health Info Libr J</source>
          <year>2018</year>
          <month>06</month>
          <volume>35</volume>
          <issue>2</issue>
          <fpage>91</fpage>
          <lpage>120</lpage>
          <pub-id pub-id-type="doi">10.1111/hir.12216</pub-id>
          <pub-id pub-id-type="medline">29729073</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="web">
          <source>Demographics of social media users and adoption in the United States</source>
          <year>2019</year>
          <access-date>2019-06-28</access-date>
          <publisher-loc>Washington</publisher-loc>
          <publisher-name>Pew Internet and American Life Project</publisher-name>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.pewinternet.org/fact-sheet/social-media/">https://www.pewinternet.org/fact-sheet/social-media/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Eysenbach</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Infodemiology and infoveillance: framework for an emerging set of public health informatics methods to analyze search, communication and publication behavior on the Internet</article-title>
          <source>J Med Internet Res</source>
          <year>2009</year>
          <volume>11</volume>
          <issue>1</issue>
          <fpage>e11</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.jmir.org/2009/1/e11/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.1157</pub-id>
          <pub-id pub-id-type="medline">19329408</pub-id>
          <pub-id pub-id-type="pii">v11i1e11</pub-id>
          <pub-id pub-id-type="pmcid">PMC2762766</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Guo</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Yu</surname>
              <given-names>Z</given-names>
            </name>
          </person-group>
          <article-title>Extracting social and community intelligence from digital footprints: an emerging research area</article-title>
          <source>Ubiquitous Intelligence and Computing</source>
          <year>2010</year>
          <publisher-loc>Berlin</publisher-loc>
          <publisher-name>Springer</publisher-name>
          <fpage>4</fpage>
          <lpage>18</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Asch</surname>
              <given-names>DA</given-names>
            </name>
            <name name-style="western">
              <surname>Rader</surname>
              <given-names>DJ</given-names>
            </name>
            <name name-style="western">
              <surname>Merchant</surname>
              <given-names>RM</given-names>
            </name>
          </person-group>
          <article-title>Mining the social mediome</article-title>
          <source>Trends Mol Med</source>
          <year>2015</year>
          <month>09</month>
          <volume>21</volume>
          <issue>9</issue>
          <fpage>528</fpage>
          <lpage>529</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/26341614"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.molmed.2015.06.004</pub-id>
          <pub-id pub-id-type="medline">26341614</pub-id>
          <pub-id pub-id-type="pii">S1471-4914(15)00121-5</pub-id>
          <pub-id pub-id-type="pmcid">PMC4662876</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>McDonald</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Malcolm</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Ramagopalan</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Syrad</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Real-world data and the patient perspective: the PROmise of social media?</article-title>
          <source>BMC Med</source>
          <year>2019</year>
          <month>01</month>
          <day>16</day>
          <volume>17</volume>
          <issue>1</issue>
          <fpage>11</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmcmedicine.biomedcentral.com/articles/10.1186/s12916-018-1247-8"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s12916-018-1247-8</pub-id>
          <pub-id pub-id-type="medline">30646913</pub-id>
          <pub-id pub-id-type="pii">10.1186/s12916-018-1247-8</pub-id>
          <pub-id pub-id-type="pmcid">PMC6334434</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wood</surname>
              <given-names>WA</given-names>
            </name>
            <name name-style="western">
              <surname>Bennett</surname>
              <given-names>AV</given-names>
            </name>
            <name name-style="western">
              <surname>Basch</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Emerging uses of patient generated health data in clinical research</article-title>
          <source>Mol Oncol</source>
          <year>2015</year>
          <month>05</month>
          <volume>9</volume>
          <issue>5</issue>
          <fpage>1018</fpage>
          <lpage>1024</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S1574-7891(14)00201-4"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.molonc.2014.08.006</pub-id>
          <pub-id pub-id-type="medline">25248998</pub-id>
          <pub-id pub-id-type="pii">S1574-7891(14)00201-4</pub-id>
          <pub-id pub-id-type="pmcid">PMC5528746</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="web">
          <article-title>Patient-focused drug development: collecting comprehensive and representative input guidance for industry, food and drug administration staff, and other stakeholders</article-title>
          <source>US Food and Drug Administration</source>
          <year>2018</year>
          <access-date>2021-01-23</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.fda.gov/downloads/drugs/guidancecomplianceregulatoryinformation/guidances/ucm610442.pdf">https://www.fda.gov/downloads/drugs/guidancecomplianceregulatoryinformation/guidances/ucm610442.pdf</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Vasconcelos Silva</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Jayasinghe</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Janda</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>What can Twitter tell us about skin cancer communication and prevention on social media?</article-title>
          <source>Dermatology</source>
          <year>2020</year>
          <volume>236</volume>
          <issue>2</issue>
          <fpage>81</fpage>
          <lpage>89</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.karger.com?DOI=10.1159/000506458"/>
          </comment>
          <pub-id pub-id-type="doi">10.1159/000506458</pub-id>
          <pub-id pub-id-type="medline">32097934</pub-id>
          <pub-id pub-id-type="pii">000506458</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wagner</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Lampos</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Cox</surname>
              <given-names>IJ</given-names>
            </name>
            <name name-style="western">
              <surname>Pebody</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>The added value of online user-generated content in traditional methods for influenza surveillance</article-title>
          <source>Sci Rep</source>
          <year>2018</year>
          <month>09</month>
          <day>18</day>
          <volume>8</volume>
          <issue>1</issue>
          <fpage>13963</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1038/s41598-018-32029-6"/>
          </comment>
          <pub-id pub-id-type="doi">10.1038/s41598-018-32029-6</pub-id>
          <pub-id pub-id-type="medline">30228285</pub-id>
          <pub-id pub-id-type="pii">10.1038/s41598-018-32029-6</pub-id>
          <pub-id pub-id-type="pmcid">PMC6143510</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hswen</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Gopaluni</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Brownstein</surname>
              <given-names>JS</given-names>
            </name>
            <name name-style="western">
              <surname>Hawkins</surname>
              <given-names>JB</given-names>
            </name>
          </person-group>
          <article-title>Using Twitter to detect psychological characteristics of self-identified persons with autism spectrum disorder: a feasibility study</article-title>
          <source>JMIR Mhealth Uhealth</source>
          <year>2019</year>
          <month>02</month>
          <day>12</day>
          <volume>7</volume>
          <issue>2</issue>
          <fpage>e12264</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://mhealth.jmir.org/2019/2/e12264/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/12264</pub-id>
          <pub-id pub-id-type="medline">30747718</pub-id>
          <pub-id pub-id-type="pii">v7i2e12264</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Malik</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Karbasian</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Hamari</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Johri</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Live, Love, Juul: user and content analysis of Twitter posts about Juul</article-title>
          <source>Am J Health Behav</source>
          <year>2019</year>
          <month>03</month>
          <day>01</day>
          <volume>43</volume>
          <issue>2</issue>
          <fpage>326</fpage>
          <lpage>336</lpage>
          <pub-id pub-id-type="doi">10.5993/AJHB.43.2.9</pub-id>
          <pub-id pub-id-type="medline">30808472</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Nielsen</surname>
              <given-names>RC</given-names>
            </name>
            <name name-style="western">
              <surname>Luengo-Oroz</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Mello</surname>
              <given-names>MB</given-names>
            </name>
            <name name-style="western">
              <surname>Paz</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Pantin</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Erkkola</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Social media monitoring of discrimination and HIV testing in Brazil, 2014-2015</article-title>
          <source>AIDS Behav</source>
          <year>2017</year>
          <month>07</month>
          <volume>21</volume>
          <issue>Suppl 1</issue>
          <fpage>114</fpage>
          <lpage>120</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/28349220"/>
          </comment>
          <pub-id pub-id-type="doi">10.1007/s10461-017-1753-2</pub-id>
          <pub-id pub-id-type="medline">28349220</pub-id>
          <pub-id pub-id-type="pii">10.1007/s10461-017-1753-2</pub-id>
          <pub-id pub-id-type="pmcid">PMC5515980</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bychkov</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Young</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Social media as a tool to monitor adherence to HIV antiretroviral therapy</article-title>
          <source>J Clin Transl Res</source>
          <year>2018</year>
          <month>12</month>
          <day>17</day>
          <volume>3</volume>
          <issue>Suppl 3</issue>
          <fpage>407</fpage>
          <lpage>410</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/30873489"/>
          </comment>
          <pub-id pub-id-type="medline">30873489</pub-id>
          <pub-id pub-id-type="pmcid">PMC6412597</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hswen</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Naslund</surname>
              <given-names>JA</given-names>
            </name>
            <name name-style="western">
              <surname>Brownstein</surname>
              <given-names>JS</given-names>
            </name>
            <name name-style="western">
              <surname>Hawkins</surname>
              <given-names>JB</given-names>
            </name>
          </person-group>
          <article-title>Monitoring online discussions about suicide among Twitter users with schizophrenia: exploratory study</article-title>
          <source>JMIR Ment Health</source>
          <year>2018</year>
          <month>12</month>
          <day>13</day>
          <volume>5</volume>
          <issue>4</issue>
          <fpage>e11483</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://mental.jmir.org/2018/4/e11483/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/11483</pub-id>
          <pub-id pub-id-type="medline">30545811</pub-id>
          <pub-id pub-id-type="pii">v5i4e11483</pub-id>
          <pub-id pub-id-type="pmcid">PMC6315229</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Tufts</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Polsky</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Volpp</surname>
              <given-names>KG</given-names>
            </name>
            <name name-style="western">
              <surname>Groeneveld</surname>
              <given-names>PW</given-names>
            </name>
            <name name-style="western">
              <surname>Ungar</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Merchant</surname>
              <given-names>RM</given-names>
            </name>
            <name name-style="western">
              <surname>Pelullo</surname>
              <given-names>AP</given-names>
            </name>
          </person-group>
          <article-title>Characterizing Tweet volume and content about common health conditions across Pennsylvania: retrospective analysis</article-title>
          <source>JMIR Public Health Surveill</source>
          <year>2018</year>
          <month>12</month>
          <day>06</day>
          <volume>4</volume>
          <issue>4</issue>
          <fpage>e10834</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://publichealth.jmir.org/2018/4/e10834/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/10834</pub-id>
          <pub-id pub-id-type="medline">30522989</pub-id>
          <pub-id pub-id-type="pii">v4i4e10834</pub-id>
          <pub-id pub-id-type="pmcid">PMC6302232</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Convertino</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Ferraro</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Blandizzi</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Tuccori</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>The usefulness of listening social media for pharmacovigilance purposes: a systematic review</article-title>
          <source>Expert Opin Drug Saf</source>
          <year>2018</year>
          <month>11</month>
          <volume>17</volume>
          <issue>11</issue>
          <fpage>1081</fpage>
          <lpage>1093</lpage>
          <pub-id pub-id-type="doi">10.1080/14740338.2018.1531847</pub-id>
          <pub-id pub-id-type="medline">30285501</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Xu</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Markson</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Costello</surname>
              <given-names>KL</given-names>
            </name>
            <name name-style="western">
              <surname>Xing</surname>
              <given-names>CY</given-names>
            </name>
            <name name-style="western">
              <surname>Demissie</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Llanos</surname>
              <given-names>AA</given-names>
            </name>
          </person-group>
          <article-title>Leveraging social media to promote public health knowledge: example of cancer awareness via Twitter</article-title>
          <source>JMIR Public Health Surveill</source>
          <year>2016</year>
          <volume>2</volume>
          <issue>1</issue>
          <fpage>e17</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://publichealth.jmir.org/2016/1/e17/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/publichealth.5205</pub-id>
          <pub-id pub-id-type="medline">27227152</pub-id>
          <pub-id pub-id-type="pii">v2i1e17</pub-id>
          <pub-id pub-id-type="pmcid">PMC4869239</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sedrak</surname>
              <given-names>MS</given-names>
            </name>
            <name name-style="western">
              <surname>Cohen</surname>
              <given-names>RB</given-names>
            </name>
            <name name-style="western">
              <surname>Merchant</surname>
              <given-names>RM</given-names>
            </name>
            <name name-style="western">
              <surname>Schapira</surname>
              <given-names>MM</given-names>
            </name>
          </person-group>
          <article-title>Cancer communication in the social media age</article-title>
          <source>JAMA Oncol</source>
          <year>2016</year>
          <month>06</month>
          <day>01</day>
          <volume>2</volume>
          <issue>6</issue>
          <fpage>822</fpage>
          <lpage>823</lpage>
          <pub-id pub-id-type="doi">10.1001/jamaoncol.2015.5475</pub-id>
          <pub-id pub-id-type="medline">26940041</pub-id>
          <pub-id pub-id-type="pii">2497877</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Pinho-Costa</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Yakubu</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Hoedebecke</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Laranjo</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Reichel</surname>
              <given-names>CP</given-names>
            </name>
            <name name-style="western">
              <surname>Colon-Gonzalez</surname>
              <given-names>MDC</given-names>
            </name>
            <name name-style="western">
              <surname>Neves</surname>
              <given-names>AL</given-names>
            </name>
            <name name-style="western">
              <surname>Errami</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Healthcare hashtag index development: identifying global impact in social media</article-title>
          <source>J Biomed Inform</source>
          <year>2016</year>
          <month>10</month>
          <volume>63</volume>
          <fpage>390</fpage>
          <lpage>399</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S1532-0464(16)30123-X"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.jbi.2016.09.010</pub-id>
          <pub-id pub-id-type="medline">27645323</pub-id>
          <pub-id pub-id-type="pii">S1532-0464(16)30123-X</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Otsuka</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Wallace</surname>
              <given-names>SA</given-names>
            </name>
            <name name-style="western">
              <surname>Chiu</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>A hashtag recommendation system for Twitter data streams</article-title>
          <source>Comput Soc Netw</source>
          <year>2016</year>
          <volume>3</volume>
          <issue>1</issue>
          <fpage>3</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/29355223"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s40649-016-0028-9</pub-id>
          <pub-id pub-id-type="medline">29355223</pub-id>
          <pub-id pub-id-type="pii">28</pub-id>
          <pub-id pub-id-type="pmcid">PMC5749337</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref37">
        <label>37</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Moreno</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>D'Angelo</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Social media intervention design: applying an affordances framework</article-title>
          <source>J Med Internet Res</source>
          <year>2019</year>
          <month>03</month>
          <day>26</day>
          <volume>21</volume>
          <issue>3</issue>
          <fpage>e11014</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2019/3/e11014/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/11014</pub-id>
          <pub-id pub-id-type="medline">30912754</pub-id>
          <pub-id pub-id-type="pii">v21i3e11014</pub-id>
          <pub-id pub-id-type="pmcid">PMC6454336</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref38">
        <label>38</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Qi</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Trang</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Doong</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Kang</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Chien</surname>
              <given-names>AL</given-names>
            </name>
          </person-group>
          <article-title>Misinformation is prevalent in psoriasis-related YouTube videos</article-title>
          <source>Dermatol Online J</source>
          <year>2016</year>
          <month>11</month>
          <day>15</day>
          <volume>22</volume>
          <issue>11</issue>
          <pub-id pub-id-type="medline">28329562</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref39">
        <label>39</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lenczowski</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Dahiya</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Psoriasis and the digital landscape: YouTube as an information source for patients and medical professionals</article-title>
          <source>J Clin Aesthet Dermatol</source>
          <year>2018</year>
          <month>03</month>
          <volume>11</volume>
          <issue>3</issue>
          <fpage>36</fpage>
          <lpage>38</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/29606999"/>
          </comment>
          <pub-id pub-id-type="medline">29606999</pub-id>
          <pub-id pub-id-type="pmcid">PMC5868783</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref40">
        <label>40</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mueller</surname>
              <given-names>SM</given-names>
            </name>
            <name name-style="western">
              <surname>Jungo</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Cajacob</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Schwegler</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Itin</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Brandt</surname>
              <given-names>O</given-names>
            </name>
          </person-group>
          <article-title>The absence of evidence is evidence of non-sense: cross-sectional study on the quality of psoriasis-related videos on YouTube and their reception by health seekers</article-title>
          <source>J Med Internet Res</source>
          <year>2019</year>
          <month>1</month>
          <day>16</day>
          <volume>21</volume>
          <issue>1</issue>
          <fpage>e11935</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.jmir.org/2019/1/e11935/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/11935</pub-id>
          <pub-id pub-id-type="medline">30664460</pub-id>
          <pub-id pub-id-type="pii">v21i1e11935</pub-id>
          <pub-id pub-id-type="pmcid">PMC6357908</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref41">
        <label>41</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Karimkhani</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Connett</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Boyers</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Quest</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Dellavalle</surname>
              <given-names>RP</given-names>
            </name>
          </person-group>
          <article-title>Dermatology on instagram</article-title>
          <source>Dermatol Online J</source>
          <year>2014</year>
          <month>07</month>
          <day>15</day>
          <volume>20</volume>
          <issue>7</issue>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://escholarship.org/uc/item/71g178w9"/>
          </comment>
          <pub-id pub-id-type="medline">25046455</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref42">
        <label>42</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Menzies</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Daly</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>McKenna</surname>
              <given-names>DB</given-names>
            </name>
          </person-group>
          <article-title>Social media and psoriasis treatment: what are people saying on Twitter?</article-title>
          <source>Br J Dermatol</source>
          <year>2019</year>
          <month>06</month>
          <volume>180</volume>
          <issue>6</issue>
          <fpage>1527</fpage>
          <lpage>1528</lpage>
          <pub-id pub-id-type="doi">10.1111/bjd.17452</pub-id>
          <pub-id pub-id-type="medline">30474251</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref43">
        <label>43</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Reuter</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Evaluating patients' perspectives on social media: the importance of clearly reporting data search, cleaning and processing</article-title>
          <source>Br J Dermatol</source>
          <year>2019</year>
          <month>07</month>
          <volume>181</volume>
          <issue>1</issue>
          <fpage>222</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/30851190"/>
          </comment>
          <pub-id pub-id-type="doi">10.1111/bjd.17868</pub-id>
          <pub-id pub-id-type="medline">30851190</pub-id>
          <pub-id pub-id-type="pmcid">PMC6602796</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref44">
        <label>44</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Shao</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Ciampaglia</surname>
              <given-names>GL</given-names>
            </name>
            <name name-style="western">
              <surname>Varol</surname>
              <given-names>O</given-names>
            </name>
            <name name-style="western">
              <surname>Yang</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Flammini</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Menczer</surname>
              <given-names>F</given-names>
            </name>
          </person-group>
          <article-title>The spread of low-credibility content by social bots</article-title>
          <source>Nat Commun</source>
          <year>2018</year>
          <month>11</month>
          <day>20</day>
          <volume>9</volume>
          <issue>1</issue>
          <fpage>4787</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/30459415"/>
          </comment>
          <pub-id pub-id-type="doi">10.1038/s41467-018-06930-7</pub-id>
          <pub-id pub-id-type="medline">30459415</pub-id>
          <pub-id pub-id-type="pii">10.1038/s41467-018-06930-7</pub-id>
          <pub-id pub-id-type="pmcid">PMC6246561</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref45">
        <label>45</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Grimme</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Preuss</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Adam</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Trautmann</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Social bots: human-like by means of human control?</article-title>
          <source>ArXiv. Preprint posted online on 6-23-2017</source>
          <year>2017</year>
          <access-date>2021-01-23</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://arxiv.org/abs/1706.07624">http://arxiv.org/abs/1706.07624</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref46">
        <label>46</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chu</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Gianvecchio</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Jajodia</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Detecting automation of Twitter accounts: are you a human, bot, or cyborg?</article-title>
          <source>IEEE Trans Dependable Secure Comput</source>
          <year>2012</year>
          <month>11</month>
          <volume>9</volume>
          <issue>6</issue>
          <fpage>811</fpage>
          <lpage>824</lpage>
          <pub-id pub-id-type="doi">10.1109/Tdsc.2012.75</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref47">
        <label>47</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mønsted</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Sapieżyński</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Ferrara</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Lehmann</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Evidence of complex contagion of information in social media: an experiment using Twitter bots</article-title>
          <source>PLoS One</source>
          <year>2017</year>
          <volume>12</volume>
          <issue>9</issue>
          <fpage>e0184148</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://dx.plos.org/10.1371/journal.pone.0184148"/>
          </comment>
          <pub-id pub-id-type="doi">10.1371/journal.pone.0184148</pub-id>
          <pub-id pub-id-type="medline">28937984</pub-id>
          <pub-id pub-id-type="pii">PONE-D-17-17034</pub-id>
          <pub-id pub-id-type="pmcid">PMC5609738</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref48">
        <label>48</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Abokhodair</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Yoo</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>McDonald</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Dissecting a social botnet: growth, content and influence in Twitter</article-title>
          <source>ArXiv. Preprint posted online on 04-13-2016</source>
          <year>2016</year>
          <access-date>2021-01-23</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://arxiv.org/abs/1604.03627">http://arxiv.org/abs/1604.03627</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref49">
        <label>49</label>
        <nlm-citation citation-type="web">
          <source>Symplur Signals: a healthcare social media analytics platform</source>
          <access-date>2019-07-09</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.symplur.com/signals/">https://www.symplur.com/signals/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref50">
        <label>50</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Katz</surname>
              <given-names>MS</given-names>
            </name>
            <name name-style="western">
              <surname>Utengen</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Anderson</surname>
              <given-names>PF</given-names>
            </name>
            <name name-style="western">
              <surname>Thompson</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>Attai</surname>
              <given-names>DJ</given-names>
            </name>
            <name name-style="western">
              <surname>Johnston</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Dizon</surname>
              <given-names>DS</given-names>
            </name>
          </person-group>
          <article-title>Disease-specific hashtags for online communication about cancer care</article-title>
          <source>JAMA Oncol</source>
          <year>2016</year>
          <month>03</month>
          <volume>2</volume>
          <issue>3</issue>
          <fpage>392</fpage>
          <lpage>394</lpage>
          <pub-id pub-id-type="doi">10.1001/jamaoncol.2015.3960</pub-id>
          <pub-id pub-id-type="medline">26539640</pub-id>
          <pub-id pub-id-type="pii">2467065</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref51">
        <label>51</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Attai</surname>
              <given-names>DJ</given-names>
            </name>
            <name name-style="western">
              <surname>Cowher</surname>
              <given-names>MS</given-names>
            </name>
            <name name-style="western">
              <surname>Al-Hamadani</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Schoger</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Staley</surname>
              <given-names>AC</given-names>
            </name>
            <name name-style="western">
              <surname>Landercasper</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Twitter social media is an effective tool for breast cancer patient education and support: patient-reported outcomes by survey</article-title>
          <source>J Med Internet Res</source>
          <year>2015</year>
          <volume>17</volume>
          <issue>7</issue>
          <fpage>e188</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.jmir.org/2015/7/e188/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.4721</pub-id>
          <pub-id pub-id-type="medline">26228234</pub-id>
          <pub-id pub-id-type="pii">v17i7e188</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref52">
        <label>52</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Utengen</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Rouholiman</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Gamble</surname>
              <given-names>JG</given-names>
            </name>
            <name name-style="western">
              <surname>Grajales</surname>
              <given-names>FJ</given-names>
            </name>
            <name name-style="western">
              <surname>Pradhan</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Staley</surname>
              <given-names>AC</given-names>
            </name>
            <name name-style="western">
              <surname>Bernstein</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Young</surname>
              <given-names>SD</given-names>
            </name>
            <name name-style="western">
              <surname>Clauson</surname>
              <given-names>KA</given-names>
            </name>
            <name name-style="western">
              <surname>Chu</surname>
              <given-names>LF</given-names>
            </name>
          </person-group>
          <article-title>Patient participation at health care conferences: engaged patients increase information flow, expand propagation, and deepen engagement in the conversation of tweets compared to physicians or researchers</article-title>
          <source>J Med Internet Res</source>
          <year>2017</year>
          <month>12</month>
          <day>17</day>
          <volume>19</volume>
          <issue>8</issue>
          <fpage>e280</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.jmir.org/2017/8/e280/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.8049</pub-id>
          <pub-id pub-id-type="medline">28818821</pub-id>
          <pub-id pub-id-type="pii">v19i8e280</pub-id>
          <pub-id pub-id-type="pmcid">PMC5579322</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref53">
        <label>53</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Parwani</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Choi</surname>
              <given-names>AD</given-names>
            </name>
            <name name-style="western">
              <surname>Lopez-Mattei</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Raza</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Narang</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Michos</surname>
              <given-names>ED</given-names>
            </name>
            <name name-style="western">
              <surname>Erwin</surname>
              <given-names>JP</given-names>
            </name>
            <name name-style="western">
              <surname>Mamas</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>Gulati</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Understanding social media: opportunities for cardiovascular medicine</article-title>
          <source>J Am Coll Cardiol</source>
          <year>2019</year>
          <month>03</month>
          <day>12</day>
          <volume>73</volume>
          <issue>9</issue>
          <fpage>1089</fpage>
          <lpage>1093</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S0735-1097(19)30108-1"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.jacc.2018.12.044</pub-id>
          <pub-id pub-id-type="medline">30846102</pub-id>
          <pub-id pub-id-type="pii">S0735-1097(19)30108-1</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref54">
        <label>54</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cumbraos-Sánchez</surname>
              <given-names>MJ</given-names>
            </name>
            <name name-style="western">
              <surname>Hermoso</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Iñiguez</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Paño-Pardo</surname>
              <given-names>JR</given-names>
            </name>
            <name name-style="western">
              <surname>Allende Bandres</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Latorre Martinez</surname>
              <given-names>MP</given-names>
            </name>
          </person-group>
          <article-title>Qualitative and quantitative evaluation of the use of Twitter as a tool of antimicrobial stewardship</article-title>
          <source>Int J Med Inform</source>
          <year>2019</year>
          <month>11</month>
          <volume>131</volume>
          <fpage>103955</fpage>
          <pub-id pub-id-type="doi">10.1016/j.ijmedinf.2019.103955</pub-id>
          <pub-id pub-id-type="medline">31487575</pub-id>
          <pub-id pub-id-type="pii">S1386-5056(19)30201-1</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref55">
        <label>55</label>
        <nlm-citation citation-type="web">
          <source>Profile Geo</source>
          <access-date>2019-07-09</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://developer.twitter.com/en/docs/tweets/enrichments/overview/profile-geo.html">https://developer.twitter.com/en/docs/tweets/enrichments/overview/profile-geo.html</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref56">
        <label>56</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kim</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Huang</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Emery</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Garbage in, garbage out: data collection, quality assessment and reporting standards for social media data use in health research, infodemiology and digital disease detection</article-title>
          <source>J Med Internet Res</source>
          <year>2016</year>
          <volume>18</volume>
          <issue>2</issue>
          <fpage>e41</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.jmir.org/2016/2/e41/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.4738</pub-id>
          <pub-id pub-id-type="medline">26920122</pub-id>
          <pub-id pub-id-type="pii">v18i2e41</pub-id>
          <pub-id pub-id-type="pmcid">PMC4788740</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref57">
        <label>57</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lui</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Vrl</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <source>langid.py: an off-the-shelf language identification tool</source>
          <access-date>2021-01-23</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.aclweb.org/anthology/P12-3005">https://www.aclweb.org/anthology/P12-3005</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref58">
        <label>58</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Allem</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Ferrara</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>The importance of debiasing social media data to better understand e-cigarette-related attitudes and behaviors</article-title>
          <source>J Med Internet Res</source>
          <year>2016</year>
          <month>08</month>
          <day>09</day>
          <volume>18</volume>
          <issue>8</issue>
          <fpage>e219</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.jmir.org/2016/8/e219/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.6185</pub-id>
          <pub-id pub-id-type="medline">27507563</pub-id>
          <pub-id pub-id-type="pii">v18i8e219</pub-id>
          <pub-id pub-id-type="pmcid">PMC5037931</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref59">
        <label>59</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ferrara</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Varol</surname>
              <given-names>O</given-names>
            </name>
            <name name-style="western">
              <surname>Davis</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Menczer</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Flammini</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <source>The rise of social bots</source>
          <year>2016</year>
          <access-date>2019-07-09</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://cacm.acm.org/magazines/2016/7/204021-the-rise-of-social-bots/fulltext">https://cacm.acm.org/magazines/2016/7/204021-the-rise-of-social-bots/fulltext</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref60">
        <label>60</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Davis</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Varol</surname>
              <given-names>O</given-names>
            </name>
            <name name-style="western">
              <surname>Ferrara</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Flammini</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Menczer</surname>
              <given-names>F</given-names>
            </name>
          </person-group>
          <article-title>BotOrNot: a system to evaluate social bots</article-title>
          <source>ArXiv. Preprint posted online on 02-02-2016</source>
          <access-date>2021-01-23</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://arxiv.org/abs/1602.00975">http://arxiv.org/abs/1602.00975</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref61">
        <label>61</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cohen</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>A coefficient of agreement for nominal scales</article-title>
          <source>Educ Psychol Meas</source>
          <year>1960</year>
          <month>04</month>
          <day>01</day>
          <volume>20</volume>
          <issue>1</issue>
          <fpage>37</fpage>
          <lpage>46</lpage>
          <pub-id pub-id-type="doi">10.1177/001316446002000104</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref62">
        <label>62</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>McHugh</surname>
              <given-names>ML</given-names>
            </name>
          </person-group>
          <article-title>Interrater reliability: the kappa statistic</article-title>
          <source>Biochem Med (Zagreb)</source>
          <year>2012</year>
          <volume>22</volume>
          <issue>3</issue>
          <fpage>276</fpage>
          <lpage>282</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.pubmedcentral.nih.gov/articlerender.fcgi?artid=3900052&#38;tool=pmcentrez&#38;rendertype=abstract"/>
          </comment>
          <pub-id pub-id-type="medline">23092060</pub-id>
          <pub-id pub-id-type="pmcid">PMC3900052</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
