Restarting R session... > # --- > # > # 01-phenotype-evaluation.R > # > # --- > > # install.packages("remotes") > # remotes::install_github("OHDSI/CohortGenerator") > # remotes::install_github("OHDSI/CohortDiagnostics") > # remotes::install_github("OHDSI/ROhdsiWebApi") > # remotes::install_github("OHDSI/OhdsiShinyModules") > > # --- > # > # libraries > # > # --- > > library(CohortGenerator) Loading required package: DatabaseConnector Loading required package: R6 > > # --- R Version --------------------- > R.Version() $platform [1] "x86_64-w64-mingw32" $arch [1] "x86_64" $os [1] "mingw32" $crt [1] "ucrt" $system [1] "x86_64, mingw32" $status [1] "" $major [1] "4" $minor [1] "3.1" $year [1] "2023" $month [1] "06" $day [1] "16" $`svn rev` [1] "84548" $language [1] "R" $version.string [1] "R version 4.3.1 (2023-06-16 ucrt)" $nickname [1] "Beagle Scouts" > # --- Java Version ------------------ > system("java -version") java version "1.8.0_261" Java(TM) SE Runtime Environment (build 1.8.0_261-b12) Java HotSpot(TM) 64-Bit Server VM (build 25.261-b12, mixed mode) [1] 0 > # ----------------------------------- > > # > # functions to get databricks token (user will be prompted for keyring password) > # > > getToken <- function () { + return ( + keyring::backend_file$new()$get( + service = "databricks", + user = "token", + keyring = "databricks_keyring" + ) + ) + } > > getUrl <- function () { + url <- "jdbc:databricks://nachc-databricks.cloud.databricks.com:443/default;transportMode=http;ssl=1;httpPath=sql/protocolv1/o/3956472157536757/0123-223459-leafy532;AuthMech=3;UID=token;PWD=" + return ( + paste(url, getToken(), sep = "") + ) + } > > # --- > # > # variables for the current execution > # > # --- > > baseUrl <- "http://localhost:8080/WebAPI" > atlasCohortIds <- c(4,5) > > connectionDetails <- DatabaseConnector::createConnectionDetails ( + dbms = "spark", + connectionString = getUrl(), + pathToDriver="D:\\_YES_2023-05-28\\workspace\\SosExamples\\_COVID\\02-data-diagnostics\\drivers\\databricks\\" + ) > > databaseId <- "covid_ohdsi" > cdmDatabaseSchema <- "covid_ohdsi" > cohortDatabaseSchema <- "covid_ohdsi" > > dataFolder <- "D:\\_YES_2023-05-28\\workspace\\SosExamples\\_COVID\\04-phenotype-evaluation\\output\\covid_ohdsi\\" > incrementalFolder <- paste("incremental_", databaseId, sep="") > > # --- > # > # phenotype evaluation implementation > # > # --- > > # set the working directory > setwd("D:\\_YES_2023-05-28\\workspace\\SosExamples\\_COVID\\04-phenotype-evaluation\\output\\demo_db\\demo_cdm\\") > > # get the cohort > cohortDefinitionSet <- ROhdsiWebApi::exportCohortDefinitionSet( + baseUrl = baseUrl, + cohortIds = atlasCohortIds, + generateStats = TRUE + ) Fetching cohortId: 4 Fetching cohortId: 5 > > # save the cohort > CohortGenerator::saveCohortDefinitionSet(cohortDefinitionSet) Cohort definition saved > > # load the cohort > # cohortDefinitionSet <- CohortGenerator::getCohortDefinitionSet() > > # View(cohortDefinitionSet) > > cohorTableNames <- getCohortTableNames(cohortTable = "cohort") > > library(CohortGenerator) > > cohortTableNames <- getCohortTableNames(cohortTable = "cohort") > > library(CohortGenerator) > > cohortTableNames <- getCohortTableNames(cohortTable = "cohort") > > createCohortTables( + connectionDetails = connectionDetails, + cohortDatabaseSchema = cohortDatabaseSchema, + incremental = TRUE + ) Connecting using Spark JDBC driver Table "cohort" already exists and in incremental mode, so not recreating it. Table "cohort_inclusion" already exists and in incremental mode, so not recreating it. Table "cohort_inclusion_result" already exists and in incremental mode, so not recreating it. Table "cohort_inclusion_stats" already exists and in incremental mode, so not recreating it. Table "cohort_summary_stats" already exists and in incremental mode, so not recreating it. Table "cohort_censor_stats" already exists and in incremental mode, so not recreating it. > > generateCohortSet( + connectionDetails = connectionDetails, + cohortDefinitionSet = cohortDefinitionSet, + cohortTableNames = cohortTableNames, + incremental = TRUE, + incrementalFolder = incrementalFolder, + cdmDatabaseSchema = cdmDatabaseSchema + ) Connecting using Spark JDBC driver Initiating cluster consisting only of main thread Skipping cohortId = '4' because it is unchanged from earlier run Skipping cohortId = '5' because it is unchanged from earlier run Generating cohort set took 0.72 secs > > library(CohortDiagnostics) Loading required package: FeatureExtraction Loading required package: Andromeda Loading required package: dplyr Attaching package: ‘dplyr’ The following objects are masked from ‘package:stats’: filter, lag The following objects are masked from ‘package:base’: intersect, setdiff, setequal, union Attaching package: ‘CohortDiagnostics’ The following object is masked from ‘package:CohortGenerator’: getCohortCounts > > executeDiagnostics( + cohortDefinitionSet = cohortDefinitionSet, + connectionDetails = connectionDetails, + cdmDatabaseSchema = cdmDatabaseSchema, + cohortDatabaseSchema = cohortDatabaseSchema, + cohortTableNames = cohortTableNames, + exportFolder = file.path(dataFolder,databaseId), + databaseId = databaseId, + incremental = TRUE, + incrementalFolder = incrementalFolder, + minCellCount = 5, + runInclusionStatistics = TRUE, + runIncludedSourceConcepts = TRUE, + runOrphanConcepts = TRUE, + runTimeSeries = TRUE, + runVisitContext = TRUE, + runBreakdownIndexEvents = FALSE, + runIncidenceRate = TRUE, + runCohortRelationship = TRUE, + runTemporalCohortCharacterization = TRUE + ) Run Cohort Diagnostics started at 2023-08-11 19:43:41.044424 - Databasename was not provided. Using CDM source table - Databasedescription was not provided. Using CDM source table The following fields found in the cohortDefinitionSet will be exported in JSON format as part of metadata field of cohort table: atlasId, generateStats, logicDescription - Unexpected fields found in table cohort - atlasId, logicDescription, generateStats. These fields will be ignored. Connecting using Spark JDBC driver Found existing record keeping file in incremental folder - CreatedDiagnostics.csv Saving database metadata Saving database metadata took 0.0505 secs Counting cohort records and subjects Counting cohorts took 0.777 secs - Censoring 0 values (0%) from cohortEntries because value below minimum - Censoring 0 values (0%) from cohortSubjects because value below minimum Found 0 of 2 (0.00%) submitted cohorts instantiated. Beginning cohort diagnostics for instantiated cohorts. Fetching inclusion statistics from files Exporting cohort concept sets to csv - Unexpected fields found in table concept_sets - databaseId. These fields will be ignored. Starting concept set diagnostics Instantiating concept sets |=====================================================================================================================| 100% Creating internal concept counts table |=====================================================================================================================| 100% Executing SQL took 25.4 secs Fetching included source concepts Skipping 2 cohorts in incremental mode. Finding orphan concepts - Finding orphan concepts for concept set 'Homeless Condition' |=====================================================================================================================| 100% Executing SQL took 14.4 secs - Finding orphan concepts for concept set 'Homeless Measurement' |=====================================================================================================================| 100% Executing SQL took 12.2 secs - Finding orphan concepts for concept set 'Homeless Observation' |=====================================================================================================================| 100% Executing SQL took 12.1 secs - Finding orphan concepts for concept set 'Homeless Procedure' |=====================================================================================================================| 100% Executing SQL took 12.2 secs - Finding orphan concepts for concept set 'Homeless Visit' |=====================================================================================================================| 100% Executing SQL took 12.1 secs - Finding orphan concepts for concept set 'Not Homeless - 2023-07-12 (ALL CONCEPTS)' |=====================================================================================================================| 100% Executing SQL took 11.8 secs - Finding orphan concepts for concept set 'Homeless - 2023-07-12 (ALL CONCEPTS)' |=====================================================================================================================| 100% Executing SQL took 11.6 secs - Finding orphan concepts for concept set 'Homeless Condition' |=====================================================================================================================| 100% Executing SQL took 12.1 secs - Finding orphan concepts for concept set 'Homeless Measurement' |=====================================================================================================================| 100% Executing SQL took 11.6 secs - Finding orphan concepts for concept set 'Homeless Observation' |=====================================================================================================================| 100% Executing SQL took 12.1 secs - Finding orphan concepts for concept set 'Homeless Procedure' |=====================================================================================================================| 100% Executing SQL took 11.6 secs - Finding orphan concepts for concept set 'Homeless Visit' |=====================================================================================================================| 100% Executing SQL took 12.6 secs - Finding orphan concepts for concept set 'Homeless - 2023-07-12 (ALL CONCEPTS)' |=====================================================================================================================| 100% Executing SQL took 12.1 secs Finding orphan concepts took 4.07 mins Running concept set diagnostics took 4.73 mins Retrieving visit context for index dates Computing incidence rates Running Incidence Rate took 0.00225 secs Computing Cohort Relationship |=====================================================================================================================| 100% Executing SQL took 5.4 secs |=====================================================================================================================| 100% Executing SQL took 6.46 secs |=====================================================================================================================| 100% Executing SQL took 5.87 secs |=====================================================================================================================| 100% Executing SQL took 5.77 secs |=====================================================================================================================| 100% Executing SQL took 5.85 secs |=====================================================================================================================| 100% Executing SQL took 5.67 secs |=====================================================================================================================| 100% Executing SQL took 6.16 secs |=====================================================================================================================| 100% Executing SQL took 5.7 secs |=====================================================================================================================| 100% Executing SQL took 5.58 secs |=====================================================================================================================| 100% Executing SQL took 6.08 secs - Running Cohort Relationship iteration with batchsize 500 from row number 1 to 2 took 1.05 mins - Computing cohort relationships took 1.05 mins Running Temporal Cohort characterization Running Temporal Cohort characterization took0.00154 secs Retrieving concept information Incremental mode, no additional concept information to export. Retrieving metadata information and writing metadata Adding results to zip file Results are ready for sharing at: D:\_YES_2023-05-28\workspace\SosExamples\_COVID\04-phenotype-evaluation\output\covid_ohdsi\covid_ohdsi/Results_covid_ohdsi.zip Computing all diagnostics took 6.47 mins Warning messages: 1: In getCdmDataSourceInformation(connection = connection, cdmDatabaseSchema = cdmDatabaseSchema) : CDM Source table does not have any records. Metadata on CDM source will be limited. 2: Unknown or uninitialised column: `isSubset`. 3: Unknown or uninitialised column: `isSubset`. 4: Unknown or uninitialised column: `isSubset`. 5: Unknown or uninitialised column: `isSubset`. 6: There were 2 warnings in `dplyr::summarise()`. The first warning was: ℹ In argument: `conceptCount = max(conceptCount)`. Caused by warning in `max()`: ! no non-missing arguments to max; returning -Inf ℹ Run dplyr::last_dplyr_warnings() to see the 1 remaining warning. >