@inproceedings{Budgen2008,
abstract = {Background: A mapping study provides a systematic and objective procedure for identifying the nature and extent of the empirical study data that is available to answer a particular research question. Such studies can also form a useful preliminary step for PhD study. Aim: We set out to assess how effective such studies have been when used for software engineering topics, and to identify the specific challenges that they present. Method: We have conducted an informal review of a number of mapping studies in software engineering, describing their main characteristics and the forms of analysis employed. Results: We examine the experiences and outcomes from six mapping studies, of which four are published. From these we note a recurring theme about the problems of classification and a preponderance of ‘gaps' in the set of empirical studies. Conclusions: We identify our challenges as improving classification guidelines, encouraging better reporting of primary studies, and argue for identifying some 'empirical grand challenges' for software engineering as a focus for the community.},
author = {<NAME>, <NAME> <NAME>},
booktitle = {Proceedings of PPIG},
isbn = {9783642021527},
pages = {195--204},
title = {{Using Mapping Studies in Software Engineering}},
volume = {2},
year = {2008}
}
@incollection{Ciolkowski2003,
abstract = {A survey is an empirical research strategy for the collection of information from heterogeneous sources. In this way, survey results often exhibit a high degree of external validity. It is complementary to other empirical research strategies such as controlled experiments, which usually have their strengths in the high internal validity of the findings. While there is a growing number of (quasi-)controlled experiments reported in the software engineering literature, few results of large scale surveys have been reported there. Hence, there is still a lack of knowledge on how to use surveys in a systematic manner for software engineering empirical research. This chapter introduces a process for preparing, conducting, and analyzing a software engineering survey. The focus of the work is on questionnaire-based surveys rather than literature surveys. The survey process is driven by practical experiences from two large-scale efforts in the review and inspection area. There are two main results from this work. First, the process itself allows researchers in empirical software engineering to follow a systematic, disciplined approach. Second, the experiences from applying the process help avoid common pitfalls that endanger both the research process and its results. We report on two (descriptive) surveys on software reviews that applied the survey process, and we present our experiences, as well as models for survey effort and duration factors derived from these experiences. {\textcopyright} Springer-Verlag Berlin Heidelberg 2003.},
author = {Ciolkowski, Marcus and Laitenberger, Oliver and <NAME> and <NAME>},
booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
doi = {10.1007/978-3-540-45143-3_7},
isbn = {978-3-540-45143-3},
pages = {104--128},
publisher = {Springer Berlin Heidelberg},
title = {{Practical Experiences in the Design and Conduct of Surveys in Empirical Software Engineering}},
volume = {2765},
year = {2003}
}
@book{Creswell2018,
address = {Los Angeles, CA, USA},
author = {Creswell, <NAME> and Creswell, <NAME>},
edition = {5th},
isbn = {9781506386706},
publisher = {SAGE Publications, Inc},
title = {{Research Design: Qualitative, Quantitative, and Mixed Methods Approaches}},
year = {2018}
}
@incollection{Easterbrook2008,
abstract = {Selecting a research method for empirical software engineering research is problematic because the benefits and challenges to using each method are not yet well catalogued. Therefore, this chapter describes a number of empirical methods available. It examines the goals of each and analyzes the types of questions each best addresses. Theoretical stances behind the methods, practical considerations in the application of the methods and data collection are also briefly reviewed. Taken together, this information provides a suitable basis for both understanding and selecting from the variety of methods applicable to empirical software engineering.},
address = {London},
author = {<NAME> and <NAME>},
booktitle = {Guide to Advanced Empirical Software Engineering},
doi = {10.1007/978-1-84800-044-5_11},
keywords = {empirical,survey,theory},
pages = {285--311},
publisher = {Springer London},
title = {{Selecting Empirical Methods for Software Engineering Research}},
year = {2008}
}
@article{Eisenhardt1989,
author = {Eisenhardt, <NAME>.},
doi = {10.5465/amr.1989.4308385},
issn = {0363-7425},
journal = {Academy of Management Review},
month = {oct},
number = {4},
pages = {532--550},
title = {{Building Theories from Case Study Research}},
volume = {14},
year = {1989}
}
@article{Falessi2018,
abstract = {{\textcopyright} 2017 Springer Science+Business Media New York[Context] Controlled experiments are an important empirical method to generate and validate theories. Many software engineering experiments are conducted with students. It is often claimed that the use of students as participants in experiments comes at the cost of low external validity while using professionals does not. [Objective] We believe a deeper understanding is needed on the external validity of software engineering experiments conducted with students or with professionals. We aim to gain insight about the pros and cons of using students and professionals in experiments. [Method] We performed an unconventional, focus group approach and a follow-up survey. First, during a session at ISERN 2014, 65 empirical researchers, including the seven authors, argued and discussed the use of students in experiments with an open mind. Afterwards, we revisited the topic and elicited experts' opinions to foster discussions. Then we derived 14 statements and asked the ISERN attendees excluding the authors, to provide their level of agreement with the statements. Finally, we analyzed the researchers' opinions and used the findings to further discuss the statements. [Results] Our survey results showed that, in general, the respondents disagreed with us about the drawbacks of professionals. We, on the contrary, strongly believe that no population (students, professionals, or others) can be deemed better than another in absolute terms. [Conclusion] Using students as participants remains a valid simplification of reality needed in laboratory contexts. It is an effective way to advance software engineering theories and technologies but, like any other aspect of study settings, should be carefully considered during the design, execution, interpretation, and reporting of an experiment. The key is to understand which developer population portion is being represented by the participants in an experiment. Thus, a proposal for describing experimental participants is put forward.},
author = {<NAME> <NAME> <NAME> M{\"{<NAME>{\"{u}}<NAME>, <NAME>},
doi = {10.1007/s10664-017-9523-3},
issn = {1382-3256},
journal = {Empirical Software Engineering},
keywords = {Experimentation,Generalization,Participants in experiments,Subjects of experiments,Threats to validity},
month = {feb},
number = {1},
pages = {452--489},
publisher = {Empirical Software Engineering},
title = {{Empirical software engineering experts on the use of students and professionals in experiments}},
volume = {23},
year = {2018}
}
@book{Feyerabend1993,
address = {London, UK},
author = {<NAME>},
edition = {3rd},
publisher = {Verso},
title = {{Against Method: Outline of an Anarchistic Theory of Knowledge}},
year = {1993}
}
@article{Garousi2019,
abstract = {Context: A Multivocal Literature Review (MLR) is a form of a Systematic Literature Review (SLR) which includes the grey literature (e.g., blog posts, videos and white papers) in addition to the published (formal) literature (e.g., journal and conference papers). MLRs are useful for both researchers and practitioners since they provide summaries both the state-of-the art and –practice in a given area. MLRs are popular in other fields and have recently started to appear in software engineering (SE). As more MLR studies are conducted and reported, it is important to have a set of guidelines to ensure high quality of MLR processes and their results. Objective: There are several guidelines to conduct SLR studies in SE. However, several phases of MLRs differ from those of traditional SLRs, for instance with respect to the search process and source quality assessment. Therefore, SLR guidelines are only partially useful for conducting MLR studies. Our goal in this paper is to present guidelines on how to conduct MLR studies in SE. Method: To develop the MLR guidelines, we benefit from several inputs: (1) existing SLR guidelines in SE, (2), a literature survey of MLR guidelines and experience papers in other fields, and (3) our own experiences in conducting several MLRs in SE. We took the popular SLR guidelines of Kitchenham and Charters as the baseline and extended/adopted them to conduct MLR studies in SE. All derived guidelines are discussed in the context of an already-published MLR in SE as the running example. Results: The resulting guidelines cover all phases of conducting and reporting MLRs in SE from the planning phase, over conducting the review to the final reporting of the review. In particular, we believe that incorporating and adopting a vast set of experience-based recommendations from MLR guidelines and experience papers in other fields have enabled us to propose a set of guidelines with solid foundations. Conclusion: Having been developed on the basis of several types of experience and evidence, the provided MLR guidelines will support researchers to effectively and efficiently conduct new MLRs in any area of SE. The authors recommend the researchers to utilize these guidelines in their MLR studies and then share their lessons learned and experiences.},
author = {<NAME>{\"{a}}ntyl{\"{a}}, <NAME>.},
doi = {10.1016/j.infsof.2018.09.006},
issn = {09505849},
journal = {Information and Software Technology},
keywords = {Evidence-based software engineering,Grey literature,Guidelines,Literature study,Multivocal literature review,Systematic literature review,Systematic mapping study},
month = {feb},
number = {September 2018},
pages = {101--121},
publisher = {Elsevier B.V.},
title = {{Guidelines for including grey literature and conducting multivocal literature reviews in software engineering}},
volume = {106},
year = {2019}
}
@inproceedings{Garousi2016,
abstract = {Systematic Literature Reviews (SLR) may not provide insight into the "state of the practice" in SE, as they do not typically include the "grey" (non-published) literature. A Multivocal Literature Review (MLR) is a form of a SLR which includes grey literature in addition to the published (formal) literature. Only a few MLRs have been published in SE so far. We aim at raising the awareness for MLRs in SE by addressing two research questions (RQs): (1) What types of knowledge are missed when a SLR does not include the multivocal literature in a SE field? and (2) What do we, as a community, gain when we include the multivocal literature and conduct MLRs? To answer these RQs, we sample a few example SLRs and MLRs and identify the missing and the gained knowledge due to excluding or including the grey literature. We find that (1) grey literature can give substantial benefits in certain areas of SE, and that (2) the inclusion of grey literature brings forward certain challenges as evidence in them is often experience and opinion based. Given these conflicting viewpoints, the authors are planning to prepare systematic guidelines for performing MLRs in SE.},
address = {New York, New York, USA},
author = {<NAME> <NAME>{\"{a}}ntyl{\"{a}}, <NAME>.},
booktitle = {Proceedings of the 20th International Conference on Evaluation and Assessment in Software Engineering - EASE '16},
doi = {10.1145/2915970.2916008},
isbn = {9781450336918},
keywords = {Empirical software engineering,Grey literature,MLR,Multivocal Literature Reviews,Research methodology,SLR,Systematic literature reviews},
pages = {1--6},
publisher = {ACM Press},
title = {{The need for multivocal literature reviews in software engineering}},
volume = {01-03-June},
year = {2016}
}
@book{Godfrey-Smith2003,
address = {Chicago, IL, USA},
author = {<NAME>},
doi = {10.7208/chicago/9780226300610.001.0001},
isbn = {978-0-226-30063-4},
publisher = {University of Chicago Press},
title = {{Theory and Reality: An Introduction to the Philosophy of Science}},
year = {2003}
}
@article{Gregor2006,
abstract = {The aim of this research essay is to examine the structural nature of theory in Information Systems. Despite the importance of theory, questions relating to its form and structure are neglected in comparison with questions relating to epistemology. The essay addresses issues of causality, explanation, prediction, and generalization that underlie an understanding of theory. A taxonomy is proposed that classifies information systems theories with respect to the manner in which four central goals are addressed: analysis, explanation, prediction, and prescription. Five interrelated types of theory are distinguished: (1) theory for analyzing, (2) theory for explaining, (3) theory for predicting, (4) theory for explaining and predicting, and (5) theory for design and action. Examples illustrate the nature of each theory type. The applicability of the taxonomy is demonstrated by classifying a sample of journal articles. The paper contributes by showing that multiple views of theory exist and by exposing the assumptions underlying different viewpoints. In addition, it is suggested that the type of theory under development can influence the choice of an epistemological approach. Support is given for the legitimacy and value of each theory type. The building of integrated bodies of theory that encompass all theory types is advocated.},
author = {<NAME>},
doi = {10.2307/25148742},
issn = {02767783},
journal = {MIS Quarterly},
keywords = {Causality,Design science,Design theory,Explanation,Generalization,Information systems discipline,Interpretivist theory,Philosophy of science,Philosophy of social sciences,Prediction,Theory,Theory structure,Theory taxonomy},
number = {3},
pages = {611},
title = {{The Nature of Theory in Information Systems}},
volume = {30},
year = {2006}
}
@article{Hevner2007,
abstract = {As a commentary to Juhani Iivari's insightful essay, I briefly analyze design science research as an embodiment of three closely related cycles of activities. The Relevance Cycle inputs requirements from the contextual envi- ronment into the research and introduces the research artifacts into environ- mental field testing. The Rigor Cycle provides grounding theories and methods along with domain experience and expertise from the foundations knowledge base into the research and adds the new knowledge generated by the research to the growing knowledge base. The central Design Cycle sup- ports a tighter loop of research activity for the construction and evaluation of design artifacts and processes. The recognition of these three cycles in a research project clearly positions and differentiates design science from other research paradigms. The commentary concludes with a claim to the pragmatic nature},
author = {Hevner, <NAME>},
isbn = {0905-0167},
issn = {09050167},
journal = {Scandinavian Journal of Information Systems},
keywords = {design cycle,design science,relevance cycle,rigor cycle},
number = {2},
pages = {87--92},
title = {{A Three Cycle View of Design Science Research}},
volume = {19},
year = {2007}
}
@article{Hevner2004,
abstract = {Two paradigms characterize much of the research in the Information Systems discipline: behavioral science and design science. The behavioral science paradigm seeks to develop and verify theories that explain or predict human or organizational behavior. The design-science paradigm seeks to extend the boundaries of human and organizational capabilities by creating new and innovative artifacts. Both paradigms are foundational to the IS discipline, positioned as it is at the confluence of people, organizations, and technology. Our objective is to describe the performance of design-science research in Information Systems via a concise conceptual framework and clear guidelines for understanding, executing, and evaluating the research. In the design-science paradigm, knowledge and understanding of a problem domain and its solution are achieved in the building and application of the designed artifact. Three recent exemplars in the research literature are used to demonstrate the application of these guidelines. We conclude with an analysis of the challenges of performing high-quality design-science research in the context of the broader IS community.},
archivePrefix = {arXiv},
arxivId = {http://dl.acm.org/citation.cfm?id=2017212.2017217},
author = {<NAME> <NAME> <NAME> Ram},
doi = {10.2307/25148625},
eprint = {/dl.acm.org/citation.cfm?id=2017212.2017217},
issn = {02767783},
journal = {MIS Quarterly},
keywords = {Information Systems research methodologies,business environment,creativity,design artifact,design science,experimental methods,search strategies,technology infrastructure},
number = {1},
pages = {75},
primaryClass = {http:},
title = {{Design Science in Information Systems Research}},
volume = {28},
year = {2004}
}
@article{Host2000,
abstract = {In many studies in software engineering students are used instead of professional software developers, although the objective is to draw conclusions valid for professional software developers. This paper presents a study where the difference between the two groups is evaluated. People from the two groups have individually carried out a non-trivial software engineering judgement task involving the assessment of howten different factors affect the lead-time of software development projects. It is found that the differences are only minor, and it is concluded that software engineering students may be used instead of professional software developers under certain conditions. These conditions are identified and described based on generally accepted criteria for validity evaluation of empirical studies.},
author = {<NAME> <NAME> <NAME>},
doi = {10.1023/A:1026586415054},
issn = {1382-3256},
journal = {Empirical Software Engineering},
number = {3},
pages = {201--214},
title = {{Using students as subjects - a comparative study of students and professionals in lead-time impact assessment}},
volume = {5},
year = {2000}
}
@inproceedings{Hove2005,
abstract = {Many phenomena related to software development are qualitative in nature. Relevant measures of such phenomena are often collected using semi-structured interviews. Such interviews involve high costs, and the quality of the collected data is related to how the interviews are conducted. Careful planning and conducting of the interviews are therefore necessary, and experiences from interview studies in software engineering should consequently be collected and analyzed to provide advice to other researchers. We have brought together experiences from 12 software engineering studies, in which a total of 280 interviews were conducted. Four areas were particularly challenging when planning and conducting these interviews; estimating the necessary effort, ensuring that the interviewer had the needed skills, ensuring good interaction between interviewer and interviewees, and using the appropriate tools and project artifacts. The paper gives advice on how to handle these areas and suggests what information about the interviews should be included when reporting studies where interviews have been used in data collection. Knowledge from other disciplines is included. By sharing experience, knowledge about the accomplishments of software engineering interviews is increased and hence, measures of high quality can be achieved},
author = {<NAME>. and <NAME>},
booktitle = {11th IEEE International Software Metrics Symposium (METRICS'05)},
doi = {10.1109/METRICS.2005.24},
isbn = {0-7695-2371-4},
issn = {15301435},
number = {Metrics},
pages = {23--23},
publisher = {IEEE},
title = {{Experiences from Conducting Semi-structured Interviews in Empirical Software Engineering Research}},
year = {2005}
}
@techreport{Jarvinen2016,
abstract = {Literature reviews play an important role in a researcher's preparation of research problem. In order to find out a gap between what we already know and what we like to know, a researcher can utilize not only concepts (Webster and Watson 2002) but also classifications in performing literature review. It is assumed that a higher order framework can be unpacked into classifications. A classification consists of classes of a dimension, and classes can be concepts (variables). It will be shown that it is possible to derive some guidelines how to improve classifications for identifying research gaps in literature review.},
address = {Tampere, Finland},
author = {J{\"{<NAME>},
institution = {University of Tampere, School of Information Sciences},
isbn = {978-952-03-0333-4},
issn = {1799-8158},
title = {{On lenses and their improvements for identifying research gaps in literature review}},
year = {2016}
}
@incollection{Jedlitschka2008,
address = {London},
author = {Jedlitschka, Andreas and Ciolkowski, Marcus and <NAME>},
booktitle = {Guide to Advanced Empirical Software Engineering},
doi = {10.1007/978-1-84800-044-5_8},
pages = {201--228},
publisher = {Springer London},
title = {{Reporting Experiments in Software Engineering}},
year = {2008}
}
@book{Johannesson2014,
abstract = {This book is an introductory text on design science, intended to support both graduate students and researchers in structuring, undertaking and presenting design science work. It builds on established design science methods as well as recent work on presenting design science studies and ethical principles for design science, and also offers novel instruments for visualizing the results, both in the form of process diagrams and through a canvas format. This work focuses on design science as applied to information systems and technology, but it also includes examples from, and perspectives of, other fields of human practice. --},
address = {Cham},
author = {<NAME> Perjons, Erik},
booktitle = {Springer International Publishing Switzerland},
doi = {10.1007/978-3-319-10632-8},
isbn = {978-3-319-10631-1},
pages = {197},
publisher = {Springer International Publishing},
title = {{An Introduction to Design Science}},
year = {2014}
}
@article{Johnson2012,
author = {<NAME>, Mathias and <NAME>},
doi = {10.1109/MS.2012.127},
issn = {0740-7459},
journal = {IEEE Software},
keywords = {engineering,explanation,prediction,science,software engineering theory,theory},
month = {sep},
number = {5},
pages = {96--96},
publisher = {IEEE},
title = {{Where's the Theory for Software Engineering?}},
volume = {29},
year = {2012}
}
@techreport{Kasunic2005,
abstract = {A survey can characterize the knowledge, attitudes, and behaviors of a large group of people through the study of a subset of them. However, to protect the validity of conclusions drawn from a survey, certain procedures must be followed throughout the process of designing, developing, and distributing the survey questionnaire. Surveys are used extensively by software and systems engineering organizations to provide insight into complex issues, assist with problem solving, and support effective decision making. This document presents a seven-stage, end-to-end process for conducting a survey.},
address = {Pittsburgh, PA},
author = {<NAME>},
institution = {Carnegie Mellon University, Software Engineering Institute},
isbn = {0780348907},
pages = {143},
title = {{Designing an Effective Survey}},
year = {2005}
}
@inproceedings{Kitchenham2004a,
abstract = {Our objective is to describe how software engineering might benefit from an evidence-based approach and to identify the potential difficulties associated with the approach. We compared the organisation and technical infrastructure supporting evidence-based medicine (EBM) with the situation in software engineering. We considered the impact that factors peculiar to software engineering (i.e. the skill factor and the lifecycle factor) would have on our ability to practice evidence-based software engineering (EBSE). EBSE promises a number of benefits by encouraging integration of research results with a view to supporting the needs of many different stakeholder groups. However, we do not currently have the infrastructure needed for widespread adoption of EBSE. The skill factor means software engineering experiments are vulnerable to subject and experimenter bias. The lifecycle factor means it is difficult to determine how technologies will behave once deployed. Software engineering would benefit from adopting what it can of the evidence approach provided that it deals with the specific problems that arise from the nature of software engineering.},
author = {<NAME>. and <NAME> <NAME>.},
booktitle = {Proceedings. 26th International Conference on Software Engineering},
doi = {10.1109/ICSE.2004.1317449},
isbn = {0-7695-2163-0},
issn = {0270-5257},
pages = {273--281},
publisher = {IEEE Comput. Soc},
title = {{Evidence-based software engineering}},
year = {2004}
}
@article{Kitchenham2004c,
abstract = {The objective of this report is to propose a guideline for systematic reviews appropriate for software engineering researchers, including PhD students. A systematic review is a means of evaluating and interpreting all available research relevant to a particular research question, topic area, or phenomenon of interest. Systematic reviews aim to present a fair evaluation of a research topic by using a trustworthy, rigorous, and auditable methodology. The guideline presented in this report was derived from three existing guidelines used by medical researchers. The guideline has been adapted to reflect the specific problems of software engineering research. The guideline covers three phases of a systematic review: planning the review, conducting the review and reporting the review. It is at a relatively high level. It does not consider the impact of question type on the review procedures, nor does it specify in detail mechanisms needed to undertake meta-analysis.},
address = {Keele, UK},
author = {<NAME>},
doi = {10.1.1.122.3308},
institution = {Software Engineering Group, Department of Computer Sciene, Keele University},
isbn = {1353-7776},
issn = {13537776},
journal = {Keele, UK, Keele University},
number = {TR/SE-0401},
pages = {28},
pmid = {15046037},
title = {{Procedures for performing systematic reviews}},
volume = {33},
year = {2004}
}
@incollection{Kitchenham2008,
address = {London},
author = {Kitchenham, <NAME>. and <NAME>.},
booktitle = {Guide to Advanced Empirical Software Engineering},
doi = {10.1007/978-1-84800-044-5_3},
pages = {63--92},
publisher = {Springer London},
title = {{Personal Opinion Surveys}},
year = {2008}
}
@techreport{Kitchenham2007,
abstract = {The objective of this report is to propose comprehensive guidelines for systematic literature reviews appropriate for software engineering researchers, including PhD students. A systematic literature review is a means of evaluating and interpreting all available research relevant to a particular research question, topic area, or phenomenon of interest. Systematic reviews aim to present a fair evaluation of a research topic by using a trustworthy, rigorous, and auditable methodology. The guidelines presented in this report were derived from three existing guidelines used by medical researchers, two books produced by researchers with social science backgrounds and discussions with researchers from other disciplines who are involved in evidence-based practice. The guidelines have been adapted to reflect the specific problems of software engineering research. The guidelines cover three phases of a systematic literature review: planning the review, conducting the review and reporting the review. They provide a relatively high level description. They do not consider the impact of the research questions on the review procedures, nor do they specify in detail the mechanisms needed to perform meta-analysis.},
address = {Keele, UK},
author = {<NAME> and <NAME>},
booktitle = {Technical Report EBSE-2007-01},
institution = {School of Computer Science and Mathematics, Keele University},
pages = {65},
title = {{Guidelines for performing Systematic Literature reviews in Software Engineering}},
year = {2007}
}
@article{Kitchenham2009,
abstract = {Background: In 2004 the concept of evidence-based software engineering (EBSE) was introduced at the ICSE04 conference. Aims: This study assesses the impact of systematic literature reviews (SLRs) which are the recommended EBSE method for aggregating evidence. Method: We used the standard systematic literature review method employing a manual search of 10 journals and 4 conference proceedings. Results: Of 20 relevant studies, eight addressed research trends rather than technique evaluation. Seven SLRs addressed cost estimation. The quality of SLRs was fair with only three scoring less than 2 out of 4. Conclusions: Currently, the topic areas covered by SLRs are limited. European researchers, particularly those at the Simula Laboratory appear to be the leading exponents of systematic literature reviews. The series of cost estimation SLRs demonstrate the potential value of EBSE for synthesising evidence and making it available to practitioners. ?? 2008 Elsevier B.V. All rights reserved.},
author = {<NAME> and {<NAME>}, <NAME>, <NAME>, <NAME>, <NAME>},
doi = {10.1016/j.infsof.2008.09.009},
isbn = {0950-5849},
issn = {09505849},
journal = {Information and Software Technology},
keywords = {Cost estimation,Evidence-based software engineering,Systematic literature review,Systematic review quality,Tertiary study},
month = {jan},
number = {1},
pages = {7--15},
title = {{Systematic literature reviews in software engineering – A systematic literature review}},
volume = {51},
year = {2009}
}
@book{Kuhn1970,
address = {Chicago, IL, USA},
author = {<NAME>.},
edition = {2nd},
isbn = {0-226-45803-2},
publisher = {University of Chicago Press},
title = {{The Structure of Scientific Revolutions}},
year = {1970}
}
@article{Langley1999,
abstract = {In this article I describe and compare a number of alternative generic strategies for the analysis oi process data, looking at the consequences oi these strategies ior emerging theories. I evaluate the strengths and weaknesses of the strategies in terms oi their capacity to generate theory that is accurate, porsimonious. general, and useful and suggest that method and theory are inextricably intertwined, that multiple strategies are oiten advisable, and that no analysis strategy will produce theory without an uncodiiiable creative leap, however small. Finally, I argue that there is room in the organizational research literature ior more openness within the academic community toward a variety of iorms oi coupling between theory and data.},
author = {<NAME>},
doi = {10.2307/259349},
issn = {03637425},
journal = {The Academy of Management Review},
month = {oct},
number = {4},
pages = {691},
title = {{Strategies for Theorizing from Process Data}},
volume = {24},
year = {1999}
}
@techreport{Mayring2014,
address = {Klagenfurt},
author = {<NAME>},
keywords = {content analysis,empirical social research,qualitative method,quantitative method,research approach,text analysis},
title = {{Qualitative content analysis: theoretical foundation, basic procedures and software solution}},
year = {2014}
}
@inproceedings{Molleri2016,
abstract = {Background: Survey is a method of research aiming to gather data from a large population of interest. Despite being extensively used in software engineering, survey-based research faces several challenges, such as selecting a representative population sample and designing the data collection instruments. Objective: This article aims to summarize the existing guidelines, supporting instruments and recommendations on how to conduct and evaluate survey-based research. Methods: A systematic search using manual search and snowballing techniques were used to identify primary studies supporting survey research in software engineering. We used an annotated review to present the findings, describing the references of interest in the research topic. Results: The summary provides a description of 15 available articles addressing the survey methodology, based upon which we derived a set of recommendations on how to conduct survey research, and their impact in the community. Conclusion: Survey-based research in software engineering has its particular challenges, as illustrated by several articles in this review. The annotated review can contribute by raising awareness of such challenges and present the proper recommendations to overcome them.},
address = {New York, New York, USA},
author = {Molleri, <NAME> <NAME> <NAME>},
booktitle = {Proceedings of the 10th ACM/IEEE International Symposium on Empirical Software Engineering and Measurement - ESEM '16},
doi = {10.1145/2961111.2962619},
isbn = {9781450344272},
issn = {19493789},
pages = {1--6},
publisher = {ACM Press},
title = {{Survey Guidelines in Software Engineering}},
volume = {08-09-Sept},
year = {2016}
}
@article{Molleri2020a,
abstract = {Context: Over the past decade Software Engineering research has seen a steady increase in survey-based studies, and there are several guidelines providing support for those willing to carry out surveys. The need for auditing survey research has been raised in the literature. Checklists have been used both to conduct and to assess different types of empirical studies, such as experiments and case studies. Objective: To operationalize the assessment of survey studies by means of a checklist. To fulfill such goal, we aim to derive a checklist from standards for survey research and further evaluate the appropriateness of the checklist in the context of software engineering research. Method: We systematically aggregated knowledge from 12 methodological studies supporting survey-based research in software engineering. We identified the key stages of the survey process and its recommended practices through thematic analysis and vote counting. We evaluated the checklist by applying it to existing surveys and analyzed the results. Thereafter, we gathered the feedback of experts (the surveys' authors) on our analysis and used the feedback to improve the survey checklist. Results: The evaluation provided insights regarding limitations of the checklist in relation to its understanding and objectivity. In particular, 19 of the 38 checklist items were improved according to the feedback received from experts. Conclusion: The proposed checklist is appropriate for auditing survey reports as well as a support tool to guide ongoing research with regard to the survey design process. A discussion on how to use the checklist and what its implications are for research practice is also provided.},
author = {Moll{\'{e}}ri, <NAME> <NAME> <NAME>},
doi = {10.1016/j.infsof.2019.106240},
issn = {09505849},
journal = {Information and Software Technology},
keywords = {Assessment,Checklist,Methodology,Survey},
month = {mar},
pages = {106240},
title = {{An empirically evaluated checklist for surveys in software engineering}},
volume = {119},
year = {2020}
}
@inproceedings{Neto2019,
abstract = {Background: In recent years, studies involving Grey Literature (GL) have been growing and attracting the attention of researchers in software engineering (SE). One of the sources of GL refers to content produced by professionals based on their practical experiences? Recent researches in the SE states that GL can complement areas of research that are not yet clearly defined in the scientific literature. In this context, the Multivocal Literature Review (MLR), a form of Systematic Literature Review (SLR) with the inclusion of GL, emerges. Goal: Provide preliminary work about the current research involving MLR studies? First, we investigate the motivation of the researchers to include GL in review studies; and second, we examine how GL was included in the studies. Method: A tertiary study was conducted to search MLR studies published between 2009 to April of 2019. Results: The main motivations for including GL in review studies are: lack of academic research on the topic, emerging research on this topic, and complementary evidence in the GL? Internet articles and white papers were the main sources of GL data used. Conclusions: The conducting of MLR studies is still in its early stages; we have identified only 12 secondary studies. The MLR studies were conducted using guidelines for performing SLRs. What we consider to be a threat to the validity of these studies, since guidelines to conduct SLR studies do not provide recommendations for quality analysis and synthesis of primary studies, including GL.},
author = {<NAME>. and Santos, <NAME>. and Endo, <NAME> and Fagundes, <NAME>.},
booktitle = {2019 ACM/IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM)},
doi = {10.1109/ESEM.2019.8870142},
isbn = {978-1-7281-2968-6},
month = {sep},
pages = {1--6},
publisher = {IEEE},
title = {{Multivocal literature reviews in software engineering: Preliminary findings from a tertiary study}},
volume = {2019-Septe},
year = {2019}
}
@book{Okasha2016,
author = {<NAME>},
doi = {10.1093/actrade/9780198745587.001.0001},
isbn = {9780198745587},
month = {jul},
publisher = {Oxford University Press},
title = {{Philosophy of Science: A Very Short Introduction}},
year = {2016}
}
@article{Peffers2007,
abstract = {JSTOR is a not-for-profit service that helps scholars, researchers, and students discover, use, and build upon a wide range of content in a trusted digital archive. We use information technology and tools to increase productivity and facilitate new forms of scholarship. For more information about JSTOR, please contact <EMAIL>.},
author = {<NAME> Tuunanen, <NAME> Rothenberger, <NAME> and Chatterjee, Samir},
doi = {10.2753/MIS0742-1222240302},
issn = {0742-1222},
journal = {Journal of Management Information Systems},
month = {dec},
number = {3},
pages = {45--77},
title = {{A Design Science Research Methodology for Information Systems Research}},
volume = {24},
year = {2007}
}
@article{Petersen2008,
abstract = {BACKGROUND: A software engineering systematic map is a defined method to build a classification scheme and structure a software engineering field of interest. The analysis of results focuses on frequencies of publications for categories within the scheme. Thereby, the coverage of the research field can be determined. Different facets of the scheme can also be combined to answer more specific research questions. OBJECTIVE: We describe how to conduct a systematic mapping study in software engineering and provide guidelines. We also compare systematic maps and systematic reviews to clarify how to chose between them. This comparison leads to a set of guidelines for systematic maps. METHOD: We have defined a systematic mapping process and applied it to complete a systematic mapping study. Furthermore, we compare systematic maps with systematic reviews by systematically analyzing existing systematic reviews. RESULTS: We describe a process for software engineering systematic mapping studies and compare it to systematic reviews. Based on this, guidelines for conducting systematic maps are defined. CONCLUSIONS: Systematic maps and reviews are different in terms of goals, breadth, validity issues and implications. Thus, they should be used complementarily and require different methods (e.g., for analysis).},
author = {<NAME> Feldt, <NAME> Mujtaba, <NAME> Mattsson, Michael},
isbn = {0-7695-2555-5},
issn = {02181940},
journal = {EASE'08 Proceedings of the 12th international conference on Evaluation and Assessment in Software Engineering},
keywords = {evidence based software engineering,systematic mapping studies,systematic reviews},
pages = {68--77},
title = {{Systematic mapping studies in software engineering}},
year = {2008}
}
@article{Petersen2015,
abstract = {Context Systematic mapping studies are used to structure a research area, while systematic reviews are focused on gathering and synthesizing evidence. The most recent guidelines for systematic mapping are from 2008. Since that time, many suggestions have been made of how to improve systematic literature reviews (SLRs). There is a need to evaluate how researchers conduct the process of systematic mapping and identify how the guidelines should be updated based on the lessons learned from the existing systematic maps and SLR guidelines. Objective To identify how the systematic mapping process is conducted (including search, study selection, analysis and presentation of data, etc.); to identify improvement potentials in conducting the systematic mapping process and updating the guidelines accordingly. Method We conducted a systematic mapping study of systematic maps, considering some practices of systematic review guidelines as well (in particular in relation to defining the search and to conduct a quality assessment). Results In a large number of studies multiple guidelines are used and combined, which leads to different ways in conducting mapping studies. The reason for combining guidelines was that they differed in the recommendations given. Conclusion The most frequently followed guidelines are not sufficient alone. Hence, there was a need to provide an update of how to conduct systematic mapping studies. New guidelines have been proposed consolidating existing findings.},
archivePrefix = {arXiv},
arxivId = {arXiv:1011.1669v3},
author = {<NAME> Vakkalanka, <NAME> <NAME>},
doi = {10.1016/j.infsof.2015.03.007},
eprint = {arXiv:1011.1669v3},
isbn = {0360-1315},
issn = {09505849},
journal = {Information and Software Technology},
keywords = {Guidelines,Software engineering,Systematic mapping studies},
month = {aug},
pages = {1--18},
pmid = {25246403},
title = {{Guidelines for conducting systematic mapping studies in software engineering: An update}},
volume = {64},
year = {2015}
}
@article{Rainer2019,
abstract = {Background: Software engineering research has a growing interest in grey literature (GL). Aim: To improve the identification of relevant and rigorous GL. Method: We develop and demonstrate heuristics to find more relevant and rigorous GL. The heuristics generate stratified samples of search and post–search datasets using a formally structured set of search keywords. Conclusion: The heuristics require further evaluation. We are developing a tool to implement the heuristics.},
author = {<NAME> and <NAME>},
doi = {10.1016/j.infsof.2018.10.007},
issn = {09505849},
journal = {Information and Software Technology},
keywords = {Grey literature review,Quality criteria,Reasoning,Search engines},
month = {feb},
pages = {231--233},
publisher = {Elsevier B.V.},
title = {{Heuristics for improving the rigour and relevance of grey literature searches for software engineering research}},
volume = {106},
year = {2019}
}
@inproceedings{Ralph2015a,
abstract = {A process theory is an explanation of how an entity changes and develops. While software engineering is fundamentally concerned with how software artifacts change and develop, little research explicitly builds and empirically evaluates software engineering process theories. This lack of theory obstructs scientific consensus by focusing the academic community on methods. Methods inevitably oversimplify and over-rationalize reality, obfuscating crucial phenomena including uncertainty, problem framing and illusory requirements. Better process theories are therefore needed to ground software engineering in empirical reality. However, poor understanding of process theory issues impedes research and publication. This paper therefore attempts to clarify the nature and types of process theories, explore their development and provide specific guidance for their empirically evaluation.},
author = {<NAME>},
booktitle = {2015 IEEE/ACM 37th IEEE International Conference on Software Engineering},
doi = {10.1109/ICSE.2015.25},
isbn = {978-1-4799-1934-5},
issn = {02705257},
keywords = {Case study,Field study,Process theory,Questionnaire,Research methodology},
month = {may},
pages = {20--31},
publisher = {IEEE},
title = {{Developing and Evaluating Software Engineering Process Theories}},
volume = {1},
year = {2015}
}
@article{Rehman2016,
author = {<NAME> and <NAME>},
journal = {International Journal of Educational Investigations},
number = {8},
pages = {51--59},
title = {{An Introduction to Research Paradigms}},
volume = {3},
year = {2016}
}
@article{Runeson2009,
abstract = {Case study is a suitable research methodology for software engineering research since it studies contemporary phenomena in its natural context. However, the understanding of what constitutes a case study varies, and hence the quality of the resulting studies. This paper aims at providing an introduction to case study methodology and guidelines for researchers conducting case studies and readers studying reports of such studies. The content is based on the authors' own experience from conducting and reading case studies. The terminology and guidelines are compiled from different methodology handbooks in other research domains, in particular social science and information systems, and adapted to the needs in software engineering. We present recommended practices for software engineering case studies as well as empirically derived and evaluated checklists for researchers and readers of case study research.},
archivePrefix = {arXiv},
arxivId = {arXiv:gr-qc/9809069v1},
author = {<NAME> H{\"{o}}st, Martin},
doi = {10.1007/s10664-008-9102-8},
eprint = {9809069v1},
isbn = {1382325615737616},
issn = {1382-3256},
journal = {Empirical Software Engineering},
keywords = {Case study,Checklists,Guidelines,Research methodology},
month = {apr},
number = {2},
pages = {131--164},
pmid = {28843849},
primaryClass = {arXiv:gr-qc},
title = {{Guidelines for conducting and reporting case study research in software engineering}},
volume = {14},
year = {2009}
}
@inproceedings{Salman2015,
abstract = {Background: Most of the experiments in software engineering (SE) employ students as subjects. This raises concerns about the realism of the results acquired through students and adaptability of the results to software industry. Aim: We compare students and professionals to understand how well students represent professionals as experimental subjects in SE research. Method: The comparison was made in the context of two test-driven development experiments conducted with students in an academic setting and with professionals in a software organization. We measured the code quality of several tasks implemented by both subject groups and checked whether students and professionals perform similarly in terms of code quality metrics. Results: Except for minor differences, neither of the subject groups is better than the other. Professionals produce larger, yet less complex, methods when they use their traditional development approach, whereas both subject groups perform similarly when they apply a new approach for the first time. Conclusion: Given a carefully scoped experiment on a development approach that is new to both students and professionals, similar performances are observed. Further investigation is necessary to analyze the effects of subject demographics and level of experience on the results of SE experiments.},
author = {<NAME> <NAME> <NAME>},
booktitle = {2015 IEEE/ACM 37th IEEE International Conference on Software Engineering},
doi = {10.1109/ICSE.2015.82},
isbn = {978-1-4799-1934-5},
issn = {02705257},
keywords = {Code quality,Empirical study,Experimentation,Test-driven development},
month = {may},
pages = {666--676},
publisher = {IEEE},
title = {{Are Students Representatives of Professionals in Software Engineering Experiments?}},
volume = {1},
year = {2015}
}
@article{Schryen2015,
abstract = {The literature review is an established research genre in many academic disciplines, including the IS discipline. Although many scholars agree that systematic literature reviews should be rigorous, few instructional texts for compiling a solid literature review, at least with regard to the IS discipline, exist. In response to this shortage, in this tutorial, I provide practical guidance for both students and researchers in the IS community who want to methodologically conduct qualitative literature reviews. The tutorial differs from other instructional texts in two regards. First, in contrast to most textbooks, I cover not only searching and synthesizing the literature but also the challenging tasks of framing the literature review, interpreting research findings, and proposing research paths. Second, I draw on other texts that provide guidelines for writing literature reviews in the IS discipline but use many examples of published literature reviews. I use an integrated example of a literature review, which guides the reader through the overall process of compiling a literature review. Keywords:},
author = {<NAME>},
doi = {10.17705/1CAIS.03712},
issn = {15293181},
journal = {Communications of the Association for Information Systems},
keywords = {Literature review,Literature synthesis,Methodology,Research agenda,Research gaps,Tutorial},
pages = {286--325},
title = {{Writing Qualitative IS Literature Reviews—Guidelines for Synthesis, Interpretation, and Guidance of Research}},
volume = {37},
year = {2015}
}
@inproceedings{Schryen2015a,
abstract = {Literature reviews play an important role in the development of knowledge. Yet, we observe a lack of theoretical underpinning of and epistemological insights into how literature reviews can contribute to knowledge creation and have actually contributed in the IS discipline. To address these theoretical and empirical research gaps, we suggest a novel epistemological model of literature reviews. This model allows us to align different contributions of literature reviews with their underlying knowledge conversions - thereby building a bridge between the previously largely unconnected fields of literature reviews and epistemology. We evaluate the appropriateness of the model by conducting an empirical analysis of 173 IS literature reviews which were published in 39 pertinent IS journals between 2000 and 2014. Based on this analysis, we derive an epistemological taxonomy of IS literature reviews, which complements previously suggested typologies.},
address = {Fort Worth, TX, USA},
author = {<NAME> and Wagner, Gerit and Benlian, Alexander},
booktitle = {Proceedings of the 36th International Conference on Information Systems},
keywords = {Literature review,Research methods/methodology,Theory of knowledge,fort worth 2015,literature review,methodology,on information systems,research methods,theory of knowledge,thirty sixth international conference},
pages = {1--22},
title = {{Theory of Knowledge for Literature Reviews: An Epistemological Model , Taxonomy and Empirical Analysis of IS Literature}},
year = {2015}
}
@incollection{Seaman2008,
abstract = {Essential Guide to Qualitative Methods in Organizational Research is an excellent resource for students and researchers in the areas of organization studies, management research and organizational psychology, bringing together in one volume the range of methods available for undertaking qualitative data collection and analysis. The volume includes 30 chapters, each focusing on a specific technique. The chapters cover traditional research methods, analysis techniques, and interventions as well as the latest developments in the field. Each chapter reviews how the method has been used in organizational research, discusses the advantages and disadvantages of using the method, and presents a case study example of the method in use. A list of further reading is supplied for those requiring additional information about a given method. The comprehensive and accessible nature of this collection will make it an essential and lasting handbook for researchers and students studying organizations.},
address = {London},
author = {Seaman, <NAME>.},
booktitle = {Guide to Advanced Empirical Software Engineering},
doi = {10.1007/978-1-84800-044-5_2},
isbn = {0761948880},
issn = {07619488},
pages = {35--62},
pmid = {50},
publisher = {Springer London},
title = {{Qualitative Methods}},
year = {2008}
}
@article{Sharp2016,
abstract = {Ethnography is a qualitative research method used to study people and cultures. It is largely adopted in disciplines outside software engineering, including different areas of computer science. Ethnography can provide an in-depth understanding of the socio-technological realities surrounding everyday software development practice, i.e., it can help to uncover not only what practitioners do, but also why they do it. Despite its potential, ethnography has not been widely adopted by empirical software engineering researchers, and receives little attention in the related literature. The main goal of this paper is to explain how empirical software engineering researchers would benefit from adopting ethnography. This is achieved by explicating four roles that ethnography can play in furthering the goals of empirical software engineering: to strengthen investigations into the social and human aspects of software engineering; to inform the design of software engineering tools; to improve method and process development; and to inform research programmes. This article introduces ethnography, explains its origin, context, strengths and weaknesses, and presents a set of dimensions that position ethnography as a useful and usable approach to empirical software engineering research. Throughout the paper, relevant examples of ethnographic studies of software practice are used to illustrate the points being made.},
author = {<NAME> Dittrich, <NAME> <NAME>, <NAME>.},
doi = {10.1109/TSE.2016.2519887},
issn = {0098-5589},
journal = {IEEE Transactions on Software Engineering},
keywords = {Design tools and techniques,computer-supported collaborative work,human factors in software design,software engineering process},
month = {aug},
number = {8},
pages = {786--804},
publisher = {IEEE},
title = {{The Role of Ethnographic Studies in Empirical Software Engineering}},
volume = {42},
year = {2016}
}
@book{Shull2008,
abstract = {Empirical studies have become an integral element of software engineering research and practice. This unique text/reference includes chapters from some of the top international empirical software engineering researchers and focuses on the practical knowledge necessary for conducting, reporting and using empirical methods in software engineering. Part 1, Research Methods and Techniques, examines the proper use of various strategies for collecting and analysing data, and the uses for which those strategies are most appropriate. Part 2, Practical Foundations, provides a discussion of several important global issues that need to be considered from the very beginning of research planning. Finally, Knowledge Creation offers insight on using a set of disparate studies to provide useful decision support. Topics and features: Offers information across a range of techniques, methods, and qualitative and quantitative issues, providing a toolkit for the reader that is applicable across the diversity of software development contexts Presents reference material with concrete software engineering examples Provides guidance on how to design, conduct, analyse, interpret and report empirical studies, taking into account the common difficulties and challenges encountered in the field Arms researchers with the information necessary to avoid fundamental risks Tackles appropriate techniques for addressing disparate studies ensuring the relevance of empirical software engineering, and showing its practical impact Describes methods that are less often used in the field, providing less conventional but still rigorous and useful ways of collecting data Supplies detailed information on topics (such as surveys) that often contain methodological errors This broad-ranging, practical guide will prove an invaluable and useful reference for practising software engineers and researchers. In addition, it will be suitable for graduate students studying empirical methods in software development. Dr. <NAME> is a senior scientist at the Fraunhofer Center for Experimental Software Engineering, Maryland, and the director of its Measurement and Knowledge Management Division. In addition, he serves as associate editor in chief of IEEE Software magazine, specializing in empirical studies. Dr. <NAME> heads the Human Computer Interaction program at the National Research Council, Canada. She has been conducting empirical research in software engineering for the past 12 years. Dr. <NAME> is currently research director of the software engineering group of the Simula Research Laboratory, Norway, which is ranked No. 3 in the world (out of 1400 institutions) in an evaluation in 2007 in the area of software and systems engineering.},
address = {London},
archivePrefix = {arXiv},
arxivId = {arXiv:1011.1669v3},
author = {<NAME> <NAME>},
booktitle = {Guide to Advanced Empirical Software Engineering},
doi = {10.1007/978-1-84800-044-5},
editor = {<NAME>.},
eprint = {arXiv:1011.1669v3},
isbn = {978-1-84800-043-8},
issn = {1098-6596},
pages = {1--388},
pmid = {6565},
publisher = {Springer London},
title = {{Guide to Advanced Empirical Software Engineering}},
year = {2008}
}
@inproceedings{Sjoberg2007,
abstract = {We present the vision that for all fields of software engineering (SE), empirical research methods should enable the development of scientific knowledge about how useful different SE technologies are for different kinds of actors, performing different kinds of activities, on different kinds of systems. It is part of the vision that such scientific knowledge will guide the development of new SE technology and is a major input to important SE decisions in industry. Major challenges to the pursuit of this vision are: more SE research should be based on the use of empirical methods; the quality, including relevance, of the studies using such methods should be increased; there should be more and better synthesis of empirical evidence; and more theories should be built and tested. Means to meet these challenges include (1) increased competence regarding how to apply and combine alternative empirical methods, (2) tighter links between academia and industry, (3) the development of common research agendas with a focus on empirical methods, and (4) more resources for empirical research.},
author = {<NAME>. and <NAME> <NAME>},
booktitle = {Future of Software Engineering (FOSE '07)},
doi = {10.1109/FOSE.2007.30},
isbn = {0-7695-2829-5},
issn = {00985589},
month = {may},
number = {1325},
pages = {358--378},
publisher = {IEEE},
title = {{The Future of Empirical Methods in Software Engineering Research}},
volume = {SE-13},
year = {2007}
}
@incollection{Sonnenberg2012,
abstract = {The central outcome of design science research (DSR) is prescriptive knowledge in the form of IT artifacts and recommendations. However, prescrip-tive knowledge is considered to have no truth value in itself. Given this assumption, the validity of DSR outcomes can only be assessed by means of descriptive knowledge to be obtained at the conclusion of a DSR process. This is reflected in the build-evaluate pattern of current DSR methodologies. Recog-nizing the emergent nature of IT artifacts this build-evaluate pattern, however, poses unfavorable implications regarding the achievement of rigor within a DSR project. While it is vital in DSR to prove the usefulness of an artifact a ri-gorous DSR process also requires justifying and validating the artifact design it-self even before it has been put into use. This paper proposes three principles for evaluating DSR artifacts which not only address the evaluation of an arti-fact's usefulness but also the evaluation of design decisions made to build an artifact. In particular, it is argued that by following these principles the prescrip-tive knowledge produced in DSR can be considered to have a truth-like value.},
author = {<NAME> <NAME>},
booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
doi = {10.1007/978-3-642-29863-9_28},
keywords = {Design science research,design theory,epistemology,evaluation},
pages = {381--397},
title = {{Evaluations in the Science of the Artificial – Reconsidering the Build-Evaluate Pattern in Design Science Research}},
year = {2012}
}
@inproceedings{Stol2013,
abstract = {There has been a growing interest in the role of theory within Software Engineering (SE) research. For several decades, researchers within the SE research community have argued that, to become a real engineering science, SE needs to develop stronger theoretical foundations. A few authors have proposed guidelines for constructing theories, building on insights from other disciplines. However, so far, much SE research is not guided by explicit theory, nor does it produce explicit theory. In this paper we argue that SE research does, in fact, show traces of theory, which we call theory fragments. We have adapted an analytical framework from the social sciences, named the Validity Network Schema (VNS), that we use to illustrate the role of theorizing in SE research. We illustrate the use of this framework by dissecting three well known research papers, each of which has had significant impact on their respective subdisciplines. We conclude this paper by outlining a number of implications for future SE research, and show how by increasing awareness and training, development of SE theories can be improved. {\textcopyright} 2013 IEEE.},
author = {<NAME> <NAME>},
booktitle = {2013 2nd SEMAT Workshop on a General Theory of Software Engineering (GTSE)},
doi = {10.1109/GTSE.2013.6613863},
isbn = {978-1-4673-6273-3},
keywords = {Software engineering research,empirical research,middle-range theory,theory building,theory fragment},
month = {may},
pages = {5--14},
publisher = {IEEE},
title = {{Uncovering theories in software engineering}},
year = {2013}
}
@article{Venable2016,
abstract = {Evaluation is a central and essential activity in conducting rigorous Design Science Research (DSR), yet there is surprisingly little guidance about designing the DSR evaluation activity beyond suggesting possible methods that could be used for evaluation. This paper extends the notable exception of the existing framework of Pries-Heje et al [11] to address this problem. The paper proposes an extended DSR evaluation framework together with a DSR evaluation design method that can guide DSR researchers in choosing an appropriate strategy for evaluation of the design artifacts and design theories that form the output from DSR. The extended DSR evaluation framework asks the DSR researcher to consider (as input to the choice of the DSR evaluation strategy) contextual factors of goals, conditions, and constraints on the DSR evaluation, e.g. the type and level of desired rigor, the type of artifact, the need to support formative development of the designed artifacts, the properties of the artifact to be evaluated, and the constraints on resources available, such as time, labor, facilities, expertise, and access to research subjects. The framework and method support matching these in the first instance to one or more DSR evaluation strategies, including the choice of ex ante (prior to artifact construction) versus ex post evaluation (after artifact construction) and naturalistic (e.g., field setting) versus artificial evaluation (e.g., laboratory setting). Based on the recommended evaluation strategy(ies), guidance is provided concerning what methodologies might be appropriate within the chosen strategy(ies).},
author = {<NAME>},
doi = {10.1057/ejis.2014.36},
isbn = {978-3-642-29862-2},
issn = {0960-085X},
journal = {European Journal of Information Systems},
keywords = {design science research,evaluation method,evaluation strategy,information,research methodology,systems evaluation},
month = {jan},
number = {1},
pages = {77--89},
title = {{FEDS: a Framework for Evaluation in Design Science Research}},
volume = {25},
year = {2016}
}
@incollection{Venable2012,
abstract = {Evaluation is a central and essential activity in conducting rigorous Design Science Research (DSR), yet there is surprisingly little guidance about designing the DSR evaluation activity beyond suggesting possible methods that could be used for evaluation. This paper extends the notable exception of the existing framework of Pries-Heje et al [11] to address this problem. The paper proposes an extended DSR evaluation framework together with a DSR evaluation design method that can guide DSR researchers in choosing an appropriate strategy for evaluation of the design artifacts and design theories that form the output from DSR. The extended DSR evaluation framework asks the DSR researcher to consider (as input to the choice of the DSR evaluation strategy) contextual factors of goals, conditions, and constraints on the DSR evaluation, e.g. the type and level of desired rigor, the type of artifact, the need to support formative development of the designed artifacts, the properties of the artifact to be evaluated, and the constraints on resources available, such as time, labor, facilities, expertise, and access to research subjects. The framework and method support matching these in the first instance to one or more DSR evaluation strategies, including the choice of ex ante (prior to artifact construction) versus ex post evaluation (after artifact construction) and naturalistic (e.g., field setting) versus artificial evaluation (e.g., laboratory setting). Based on the recommended evaluation strategy(ies), guidance is provided concerning what methodologies might be appropriate within the chosen strategy(ies).},
author = {<NAME>},
booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
doi = {10.1007/978-3-642-29863-9_31},
keywords = {Design Science Research,Evaluation Method,Evaluation Strategy,Information Systems Evaluation,Research Methodology},
pages = {423--438},
title = {{A Comprehensive Framework for Evaluation in Design Science Research}},
year = {2012}
}
@book{Wieringa2014,
abstract = {Abstract Design scientists have to balance the demands of methodological rigor that they share with purely curiosity-driven scientists, with the demands of practical utility that they share with utility-driven engineers. Balancing these conflicting demands can be ... $\backslash$n},
address = {Berlin, Heidelberg},
author = {Wieringa, <NAME>.},
doi = {10.1007/978-3-662-43839-8},
isbn = {978-3-662-43838-1},
keywords = {Design Science},
pages = {493},
publisher = {Springer Berlin Heidelberg},
title = {{Design Science Methodology for Information Systems and Software Engineering}},
year = {2014}
}
@incollection{Wohlin2013,
abstract = {The dependence on quality software in all areas of life is what makes software engineering a key discipline for todays society. Thus, over the last few decades it has been increasingly recognized that it is particularly important to demonstrate the value of software engineering methods in real-world environments, a task which is the focus of empirical software engineering. One of the leading protagonists of this discipline worldwide is Prof. Dr. Dr. h.c. <NAME>, who dedicated his entire career to empirical software engineering. For his many important contributions to the field he has received numerous awards and recognitions, including the U.S. National Science Foundations Presidential Young Investigator Award and the Cross of the Order of Merit of the Federal Republic of Germany. He is a Fellow of both the ACM and the IEEE Computer Society. This book, published in honor of his 60th birthday, is dedicated to Dieter Rombach and his contributions to software engineering in general, as well as to empirical software engineering in particular. This book presents invited contributions from a number of the most internationally renowned software engineering researchers like <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, and, of course, by <NAME> himself. Several key experts from the Fraunhofer IESE, the institute founded and led by <NAME>, also contributed to the book. The contributions summarize some of the most important trends in software engineering today and outline a vision for the future of the field. The book is structured into three main parts. The first part focuses on the classical foundations of software engineering, such as notations, architecture, and processes, while the second addresses empirical software engineering in particular as the core field of Dieter Rombachs contributions. Finally, the third part discusses a broad vision for the future of software engineering.},
address = {Berlin, Heidelberg},
author = {<NAME>},
booktitle = {Perspectives on the Future of Software Engineering},
doi = {10.1007/978-3-642-37395-4_10},
isbn = {9783642373954},
pages = {145--157},
publisher = {Springer Berlin Heidelberg},
title = {{An Evidence Profile for Software Engineering Research and Practice}},
volume = {9783642373},
year = {2013}
}
@inproceedings{Wohlin2014,
abstract = {Background: Systematic literature studies have become common in software engineering, and hence it is important to understand how to conduct them efficiently and reliably. Objective: This paper presents guidelines for conducting literature reviews using a snowballing approach, and they are illustrated and evaluated by replicating a published systematic literature review. Method: The guidelines are based on the experience from conducting several systematic literature reviews and experimenting with different approaches. Results: The guidelines for using snowballing as a way to search for relevant literature was successfully applied to a systematic literature review. Conclusions: It is concluded that using snowballing, as a first search strategy, may very well be a good alternative to the use of database searches. Copyright 2014 ACM.},
address = {New York, New York, USA},
author = {<NAME>},
booktitle = {Proceedings of the 18th International Conference on Evaluation and Assessment in Software Engineering - EASE '14},
doi = {10.1145/2601248.2601268},
isbn = {9781450324762},
keywords = {Replication,Snowball search,Snowballing,Systematic literature review,Systematic mapping studies},
pages = {1--10},
publisher = {ACM Press},
title = {{Guidelines for snowballing in systematic literature studies and a replication in software engineering}},
year = {2014}
}
@incollection{Wohlin2003,
author = {<NAME> <NAME>},
booktitle = {Esernet},
doi = {10.1007/978-3-540-45143-3_2},
isbn = {3-540-40672-7},
keywords = {dblp},
pages = {7--23},
title = {{Empirical Research Methods in Software Engineering}},
volume = {2765},
year = {2003}
}
@book{Wohlin2012,
abstract = {Empirical software engineering research can be organized in several ways, including experiments, cases studies, and surveys. Experiments sample over the variables, trying to represent all possible cases; cases studies sample from the variables, representing only the typical cases(s). Every case study or experiment should have a hypothesis to express the desired result. The experimental design is especially important because it identifies key variables and their relationships. The design uses balancing, blocking, and local control to help minimize error. Analysis techniques depend on the design, the distribution of the data, and the type of investigation being carried out. Different techniques allow us to look at variable interaction and to look at combinations of effects. Using a technique similar to a board game, we can determine when we have enough evidence to demonstrate clear relationships among variables. {\textcopyright} 1997 Academic Press Inc.},
address = {Berlin, Heidelberg},
author = {<NAME> Runeson, <NAME>{\"{o}}st, <NAME> Ohlsson, <NAME>. and Regnell, Bj{\"{<NAME>{\'{e}}n, Anders},
booktitle = {Experimentation in Software Engineering},
doi = {10.1007/978-3-642-29044-2},
isbn = {978-3-642-29043-5},
pages = {1--236},
publisher = {Springer Berlin Heidelberg},
title = {{Experimentation in Software Engineering}},
volume = {9783642290},
year = {2012}
}
@inproceedings{Zhou2016a,
abstract = {—Context: The assessment of Threats to Validity (TTVs) is critical to secure the quality of empirical studies in Software Engineering (SE). In the recent decade, Systematic Literature Review (SLR) was becoming an increasingly important empirical research method in SE as it was able to provide the strongest evidence. One of the mechanisms of insuring the level of scientific value in the findings of an SLR is to rigorously assess its validity. Hence, it is necessary to realize the status quo and issues of TTVs of SLRs in SE. Objective: This study aims to investigate the-state-of-the-practice of TTVs of the SLRs published in SE, and further support SE researchers to improve the assessment and strategies against TTVs in order to increase the quality of SLRs in SE. Method: We conducted a tertiary study by reviewing the SLRs in SE that report the assessment of TTVs. Results: We identified 316 SLRs published from 2004 to the first half of 2015, in which TTVs are discussed. The issues associated to TTVs were also summarized and categorized. Conclusion: The common TTVs related to SLR research, such as internal validity and reliability, were thoroughly discussed in most SLRs. The threats to construct validity and external validity drew less attention. Moreover, there are few strategies and tactics being reported to cope with the various TTVs.},
address = {Hamilton, New Zealand},
author = {<NAME> <NAME> <NAME>},
booktitle = {2016 23rd Asia-Pacific Software Engineering Conference (APSEC)},
doi = {10.1109/APSEC.2016.031},
isbn = {978-1-5090-5575-3},
keywords = {Evidence-Based Software Engineering,Systematic (Literature) Review,Threats to Validity},
pages = {153--160},
publisher = {IEEE},
title = {{A Map of Threats to Validity of Systematic Literature Reviews in Software Engineering}},
year = {2016}
}
