/*
 * Copyright (c) 2018-2020, NVIDIA CORPORATION.  All rights reserved.
 *
 * NVIDIA CORPORATION and its licensors retain all intellectual property
 * and proprietary rights in and to this software, related documentation
 * and any modifications thereto.  Any use, reproduction, disclosure or
 * distribution of this software and related documentation without an express
 * license agreement from NVIDIA CORPORATION is strictly prohibited.
 */

/**
 * @file
 * <b>NvSIPL Camera sample application. </b>
 *
 * @b Description: This application demonstrates the usage of NvSIPL APIs to,
 *  1. Select a pre-defined platform configuration using Query APIs.
 *  2. Create and configure camera pipelines using Camera APIs.
 *  3. Create an NvSIPL client to consume images using NvSIPL Client APIs.
 *  4. Implement callbacks to receive and process the outputs a pipeline using Client APIs.
 */

/* STL Headers */
#include <unistd.h>
#include <cstring>
#include <iostream>
#include <csignal>
#include <thread>
#include <chrono>
#include <ctime>
#include <atomic>
#include <cmath>
#include <pthread.h>

/* NvSIPL Headers */
#include "NvSIPLVersion.hpp" // Version
#include "NvSIPLTrace.hpp" // Trace
#include "NvSIPLCommon.hpp" // Common
#include "NvSIPLQuery.hpp" // Query
#include "NvSIPLQueryTrace.hpp" // Query Trace
#include "NvSIPLCamera.hpp" // Camera
#include "NvSIPLPipelineMgr.hpp" // Pipeline manager
#include "NvSIPLClient.hpp" // Client

#include "CNvSIPLMaster.hpp"
#include "CNvSIPLMasterNvSci.hpp"
#include "CFileReader.hpp"
#include "CFileWriter.hpp"
#include "CNvSIPLConsumer.hpp"
#include "CCmdLineParser.hpp"
#include "CComposite.hpp"
#include "CCompositeNvSci.hpp"
#include "CSamplePlugin.hpp"
#include "CAutoRecovery.hpp"

#define MAX_NUM_SENSORS (16u)
#define MAX_NUM_OUTPUTS_PER_SENSOR (3u)
#define SECONDS_PER_HOUR (3600u)
#define SECONDS_PER_ITERATION (2)

/* Quit flag. */
std::atomic<bool> bQuit;

/* Error reported by library via callbacks */
std::unique_ptr<CAutoRecovery> m_upAutoRecovery {nullptr};

/** Signal handler.*/
static void SigHandler(int signum)
{
    LOG_WARN("Received signal: %u. Quitting\n", signum);
    signal(SIGINT, SIG_IGN);
    signal(SIGTERM, SIG_IGN);
    signal(SIGQUIT, SIG_IGN);
    signal(SIGHUP, SIG_IGN);

    bQuit = true;

    signal(SIGINT, SIG_DFL);
    signal(SIGTERM, SIG_DFL);
    signal(SIGQUIT, SIG_DFL);
    signal(SIGHUP, SIG_DFL);
}

/** Sets up signal handler.*/
static void SigSetup(void)
{
    struct sigaction action { };
    action.sa_handler = SigHandler;

    sigaction(SIGINT, &action, NULL);
    sigaction(SIGTERM, &action, NULL);
    sigaction(SIGQUIT, &action, NULL);
    sigaction(SIGHUP, &action, NULL);
}

class CPipelineNotificationHandler : public NvSIPLPipelineNotifier
{
public:
    //! Notifier function
    void OnEvent(NotificationData &oNotificationData) override final
    {
        switch (oNotificationData.eNotifType) {
        case NOTIF_INFO_ICP_PROCESSING_DONE:
            LOG_INFO("Pipeline: %u, NOTIF_INFO_ICP_PROCESSING_DONE\n", oNotificationData.uIndex);
            break;
        case NOTIF_INFO_ISP_PROCESSING_DONE:
            LOG_INFO("Pipeline: %u, NOTIF_INFO_ISP_PROCESSING_DONE\n", oNotificationData.uIndex);
            break;
        case NOTIF_INFO_ACP_PROCESSING_DONE:
            LOG_INFO("Pipeline: %u, NOTIF_INFO_ACP_PROCESSING_DONE\n", oNotificationData.uIndex);
            break;
        case NOTIF_INFO_CDI_PROCESSING_DONE:
            LOG_INFO("Pipeline: %u, NOTIF_INFO_CDI_PROCESSING_DONE\n", oNotificationData.uIndex);
            break;
        case NOTIF_WARN_ICP_FRAME_DROP:
            LOG_WARN("Pipeline: %u, NOTIF_WARN_ICP_FRAME_DROP\n", oNotificationData.uIndex);
            m_uNumFrameDrops[oNotificationData.uIndex]++;
            break;
        case NOTIF_WARN_ICP_FRAME_DISCONTINUITY:
            LOG_WARN("Pipeline: %u, NOTIF_WARN_ICP_FRAME_DISCONTINUITY\n", oNotificationData.uIndex);
            break;
        case NOTIF_WARN_ICP_CAPTURE_TIMEOUT:
            LOG_WARN("Pipeline: %u, NOTIF_WARN_ICP_CAPTURE_TIMEOUT\n", oNotificationData.uIndex);
            break;
        case NOTIF_ERROR_ICP_BAD_INPUT_STREAM:
            LOG_ERR("Pipeline: %u, NOTIF_ERROR_ICP_BAD_INPUT_STREAM\n", oNotificationData.uIndex);
            if (m_upAutoRecovery == nullptr) {
                m_bInError = true; // Treat this as fatal error only if link recovery is not enabled.
            }
            break;
        case NOTIF_ERROR_ICP_CAPTURE_FAILURE:
            LOG_ERR("Pipeline: %u, NOTIF_ERROR_ICP_CAPTURE_FAILURE\n", oNotificationData.uIndex);
            m_bInError = true;
            break;
        case NOTIF_ERROR_ICP_EMB_DATA_PARSE_FAILURE:
            LOG_ERR("Pipeline: %u, NOTIF_ERROR_ICP_EMB_DATA_PARSE_FAILURE\n", oNotificationData.uIndex);
            m_bInError = true;
            break;
        case NOTIF_ERROR_ISP_PROCESSING_FAILURE:
            LOG_ERR("Pipeline: %u, NOTIF_ERROR_ISP_PROCESSING_FAILURE\n", oNotificationData.uIndex);
            m_bInError = true;
            break;
        case NOTIF_ERROR_ACP_PROCESSING_FAILURE:
            LOG_ERR("Pipeline: %u, NOTIF_ERROR_ACP_PROCESSING_FAILURE\n", oNotificationData.uIndex);
            m_bInError = true;
            break;
        case NOTIF_ERROR_CDI_SET_SENSOR_CTRL_FAILURE:
            LOG_ERR("Pipeline: %u, NOTIF_ERROR_CDI_SET_SENSOR_CTRL_FAILURE\n", oNotificationData.uIndex);
            if (m_upAutoRecovery == nullptr) {
                m_bInError = true; // Treat this as fatal error only if link recovery is not enabled.
            }
            break;
        case NOTIF_ERROR_DESER_LINK_FAILURE:
            LOG_ERR("Pipeline: %u, NOTIF_ERROR_DESER_LINK_FAILURE\n", oNotificationData.uIndex);
            if (m_upAutoRecovery != nullptr) {
                m_upAutoRecovery->OnLinkFailure(oNotificationData.uIndex);
            } else {
                m_bInError = true; // Treat this as fatal error only if link recovery is not enabled.
            }
            break;
        case NOTIF_ERROR_INTERNAL_FAILURE:
            m_bInError = true;
            LOG_ERR("Pipeline: %u, NOTIF_ERROR_INTERNAL_FAILURE\n", oNotificationData.uIndex);
            break;
        default:
            LOG_WARN("Pipeline: %u, Unknown/Invalid notification\n", oNotificationData.uIndex);
            break;
        }

        return;
    }

    //! Returns true to pipeline encountered any fatal error.
    bool IsPipelineInError(void) {
        return m_bInError;
    }

    //! Get number of frame drops for a given sensor
    uint32_t GetNumFrameDrops(uint32_t uSensor) {
        if (uSensor >= MAX_NUM_SENSORS) {
            LOG_ERR("Invalid index used to request number of frame drops\n");
            return -1;
        }
        return m_uNumFrameDrops[uSensor];
    }

    //! Reset frame drop counter
    void ResetFrameDropCounter(void) {
        for (auto i = 0u; i < MAX_NUM_SENSORS; i++) {
            m_uNumFrameDrops[i] = 0u;
        }
    }

private:
    uint32_t m_uNumFrameDrops[MAX_NUM_SENSORS] = {0u};
    bool m_bInError = false;
};

int main(int argc, char *argv[])
{
    pthread_setname_np(pthread_self(), "Main");

    bQuit = false;

    LOG_INFO("Checking SIPL version\n");
    NvSIPLVersion oVer;
    NvSIPLGetVersion(oVer);

    LOG_INFO("NvSIPL library version: %u.%u.%u\n", oVer.uMajor, oVer.uMinor, oVer.uPatch);
    LOG_INFO("NVSIPL header version: %u %u %u\n", NVSIPL_MAJOR_VER, NVSIPL_MINOR_VER, NVSIPL_PATCH_VER);
    if (oVer.uMajor != NVSIPL_MAJOR_VER || oVer.uMinor != NVSIPL_MINOR_VER || oVer.uPatch != NVSIPL_PATCH_VER) {
        LOG_ERR("NvSIPL library and header version mismatch\n");
    }

    // INvSIPLQuery
    auto pQuery = INvSIPLQuery::GetInstance();
    CHK_PTR_AND_RETURN(pQuery, "INvSIPLQuery::GetInstance");

    auto status = pQuery->ParseDatabase();
    CHK_STATUS_AND_RETURN(status, "INvSIPLQuery::ParseDatabase");

    LOG_INFO("Parsing command line arguments\n");
    CCmdLineParser cmdline;
    auto ret = cmdline.Parse(argc, argv);
    if (ret != 0) {
        // No need to print any error, Parse() would have printed error.
        return -1;
    }

    if (cmdline.verbosity >= INvSIPLTrace::LevelInfo) {
        cmdline.PrintArgs();
    }

    std::unique_ptr<CComposite> upComposite(nullptr);
    if (cmdline.uDisplay != (uint32_t)-1) {
        LOG_INFO("Creating compositor\n");
        if (cmdline.bNvSci) {
            upComposite.reset(new CCompositeNvSci());
            CHK_PTR_AND_RETURN(upComposite, "NvSci compositor creation");
        } else {
            upComposite.reset(new CComposite());
            CHK_PTR_AND_RETURN(upComposite, "Compositor creation");
        }
    }

    // Set verbosity level
    LOG_INFO("Setting verbosity level: %u\n", cmdline.verbosity);
    INvSIPLQueryTrace::GetInstance()->SetLevel((INvSIPLQueryTrace::TraceLevel)cmdline.verbosity);
    INvSIPLTrace::GetInstance()->SetLevel((INvSIPLTrace::TraceLevel)cmdline.verbosity);
    CLogger::GetInstance().SetLogLevel((CLogger::LogLevel) cmdline.verbosity);

    LOG_INFO("Setting up signal handler\n");
    SigSetup();

    if (cmdline.sTestConfigFile != "") {
        status = pQuery->ParseJsonFile(cmdline.sTestConfigFile);
        CHK_STATUS_AND_RETURN(status, "INvSIPLQuery::ParseJsonFile");
    }

    LOG_INFO("Getting platform configuration for %s\n", cmdline.sConfigName.c_str());
    PlatformCfg oPlatformCfg;
    status = pQuery->GetPlatformCfg(cmdline.sConfigName, oPlatformCfg);
    CHK_STATUS_AND_RETURN(status, "INvSIPLQuery::GetPlatformCfg");

    // Apply mask
    if (cmdline.vMasks.size() != 0) {
        LOG_INFO("Setting link masks\n");
        status = pQuery->ApplyMask(oPlatformCfg, cmdline.vMasks);
        CHK_STATUS_AND_RETURN(status, "INvSIPLQuery::ApplyMask");
    }

    // Check if file mode.
    // If so we need to verify/update device block info and create fileReader
    std::unique_ptr<CFileReader> fileReader(nullptr);
    if (!cmdline.vInputRawFiles.empty()) {
        if (oPlatformCfg.numDeviceBlocks != 1) {
            LOG_ERR("Only one device block is supported in simulator mode. Please correct mask.\n");
            return -1;
        }
        if (oPlatformCfg.deviceBlockList[0].numCameraModules != 1) {
            LOG_ERR("Only one camera module is supported in simulator mode. Please correct mask.\n");
            return -1;
        }

        oPlatformCfg.deviceBlockList[0].isSimulatorModeEnabled = true;
        // Create new file reader
        fileReader.reset(new CFileReader());
        CHK_PTR_AND_RETURN(fileReader, "FileReader creation");

        // Initialize the feeder
        const auto& vcinfo = oPlatformCfg.deviceBlockList[0].cameraModuleInfoList[0].sensorInfo.vcInfo;
        auto status = fileReader->Init(cmdline.vInputRawFiles, vcinfo, &bQuit);
        CHK_STATUS_AND_RETURN(status, "FileReader initialization");
    }

    // Warn if file dumping is requested with more than one output enabled
    if (cmdline.sPrefix != "") {
        uint32_t uNumOutputsPerSensor = (uint32_t)(!cmdline.bDisableRaw)
                                      + (uint32_t)(!cmdline.bDisableISP0)
                                      + (uint32_t)(!cmdline.bDisableISP1);
        if (!((oPlatformCfg.numDeviceBlocks == 1)
              && (oPlatformCfg.deviceBlockList[0].numCameraModules == 1)
              && (uNumOutputsPerSensor == 1))) {
            LOG_WARN("More than one output is requested for file dump. Frame drops may occur.\n");
        }
    }

    // Check if slave mode is enabled.
    // If so we need to modify device block info.
    for (auto d = 0u; d != oPlatformCfg.numDeviceBlocks; d++ ) {
        if (cmdline.bEnableSlave) {
            oPlatformCfg.deviceBlockList[d].isSlaveModeEnabled = true;
        }
    }

    // Check platform configuration and pipeline configuration compatibility.
    for (auto d = 0u; d != oPlatformCfg.numDeviceBlocks; d++ ) {
        auto db = oPlatformCfg.deviceBlockList[d];
        for (auto m = 0u; m != db.numCameraModules; m++) {
            auto module = db.cameraModuleInfoList[m];
            auto sensor = module.sensorInfo;
            if (sensor.isTPGEnabled and (!cmdline.bDisableISP0 or !cmdline.bDisableISP1)) {
                LOG_ERR("Cannot enable ISP output in TPG mode.\n");
                return -1;
            }
        }
    }

    // upNotificationHandler needs to outlive INvSIPLCamera so it must be created before upMaster
    std::unique_ptr<CPipelineNotificationHandler> upNotificationHandler(nullptr);
    upNotificationHandler.reset(new CPipelineNotificationHandler());
    CHK_PTR_AND_RETURN(upNotificationHandler, "Notification handler creation");

    LOG_INFO("Creating camera master\n");
    std::unique_ptr<CNvSIPLMaster> upMaster(nullptr);
    if (cmdline.bNvSci) {
        upMaster.reset(new CNvSIPLMasterNvSci());
        CHK_PTR_AND_RETURN(upMaster, "NvSci camera master creation");
    } else {
        upMaster.reset(new CNvSIPLMaster());
        CHK_PTR_AND_RETURN(upMaster, "Camera master creation");
    }

    NvSciBufModule bufModule = nullptr;
    NvSciSyncModule syncModule = nullptr;

    LOG_INFO("Setting up master\n");
    status = upMaster->Setup(&oPlatformCfg, &bufModule, &syncModule);
    CHK_STATUS_AND_RETURN(status, "Master setup");

    if (upComposite != nullptr) {
        LOG_INFO("Initializing compositor\n");
        SIPLStatus status = NVSIPL_STATUS_ERROR;
        status = upComposite->Init(cmdline.uDisplay,
                                   cmdline.uDispWinID,
                                   cmdline.uDispWinDepth,
                                   cmdline.bRectSet ? &cmdline.oDispRect : nullptr,
                                   bufModule,
                                   syncModule);
        CHK_STATUS_AND_RETURN(status, "Compositor initialization");
    }

    // Enable following outputs for all sensors
    std::vector<INvSIPLClient::ConsumerDesc::OutputType> eOutputList;
    if (!cmdline.bDisableRaw) {
        eOutputList.push_back(INvSIPLClient::ConsumerDesc::OutputType::ICP);
    }

    if (!cmdline.bDisableISP0) {
        eOutputList.push_back(INvSIPLClient::ConsumerDesc::OutputType::ISP0);
    }

    if (!cmdline.bDisableISP1) {
        eOutputList.push_back(INvSIPLClient::ConsumerDesc::OutputType::ISP1);
    }

    LOG_INFO("Creating consumers\n");
    vector<unique_ptr<CNvSIPLConsumer>> vupConsumers;
    NvSciStreamBlock consumer[MAX_NUM_SENSORS][MAX_NUM_OUTPUTS_PER_SENSOR];
    NvSciStreamBlock *consumerUpstream[MAX_NUM_SENSORS][MAX_NUM_OUTPUTS_PER_SENSOR] = {{nullptr}};
    NvSciStreamBlock queue[MAX_NUM_SENSORS][MAX_NUM_OUTPUTS_PER_SENSOR];
    // for each sensor
    for (auto d = 0u; d != oPlatformCfg.numDeviceBlocks; d++) {
        auto db = oPlatformCfg.deviceBlockList[d];
        for (auto m = 0u; m != db.numCameraModules; m++) {
            auto module = db.cameraModuleInfoList[m];
            NvSIPLPipelineCfg pipelineCfg;

            auto sensor = module.sensorInfo;
            uint32_t uSensor = sensor.id;

            auto isRawSensor = false;
            if ((sensor.vcInfo.inputFormat == NVMEDIA_IMAGE_CAPTURE_INPUT_FORMAT_TYPE_RAW6) or
                (sensor.vcInfo.inputFormat == NVMEDIA_IMAGE_CAPTURE_INPUT_FORMAT_TYPE_RAW7) or
                (sensor.vcInfo.inputFormat == NVMEDIA_IMAGE_CAPTURE_INPUT_FORMAT_TYPE_RAW8) or
                (sensor.vcInfo.inputFormat == NVMEDIA_IMAGE_CAPTURE_INPUT_FORMAT_TYPE_RAW10) or
                (sensor.vcInfo.inputFormat == NVMEDIA_IMAGE_CAPTURE_INPUT_FORMAT_TYPE_RAW12) or
                (sensor.vcInfo.inputFormat == NVMEDIA_IMAGE_CAPTURE_INPUT_FORMAT_TYPE_RAW14) or
                (sensor.vcInfo.inputFormat == NVMEDIA_IMAGE_CAPTURE_INPUT_FORMAT_TYPE_RAW16) or
                (sensor.vcInfo.inputFormat == NVMEDIA_IMAGE_CAPTURE_INPUT_FORMAT_TYPE_RAW20)) {
                isRawSensor = true;
            }

            // for each output
            for (INvSIPLClient::ConsumerDesc::OutputType eOutput : eOutputList) {
                // Create NvSIPL consumer using client descriptor
                LOG_INFO("Creating consumer for output:%u of sensor:%u\n", eOutput, uSensor);
                auto upCons = unique_ptr<CNvSIPLConsumer>(new CNvSIPLConsumer());
                CHK_PTR_AND_RETURN(upCons, "Consumer creation");

                // Register the consumer with the compositor as source
                auto uID = -1u;
                if (upComposite != nullptr) {
                    auto grpIndex = 0;
                    switch (db.csiPort) {
                        default:
                        case NVMEDIA_IMAGE_CAPTURE_CSI_INTERFACE_TYPE_CSI_AB:
                        case NVMEDIA_IMAGE_CAPTURE_CSI_INTERFACE_TYPE_CSI_A:
                        case NVMEDIA_IMAGE_CAPTURE_CSI_INTERFACE_TYPE_CSI_B:
                            grpIndex = 0;
                            break;
                        case NVMEDIA_IMAGE_CAPTURE_CSI_INTERFACE_TYPE_CSI_CD:
                        case NVMEDIA_IMAGE_CAPTURE_CSI_INTERFACE_TYPE_CSI_C:
                        case NVMEDIA_IMAGE_CAPTURE_CSI_INTERFACE_TYPE_CSI_D:
                            grpIndex = 1;
                            break;
                        case NVMEDIA_IMAGE_CAPTURE_CSI_INTERFACE_TYPE_CSI_EF:
                        case NVMEDIA_IMAGE_CAPTURE_CSI_INTERFACE_TYPE_CSI_E:
                        case NVMEDIA_IMAGE_CAPTURE_CSI_INTERFACE_TYPE_CSI_F:
                            grpIndex = 2;
                            break;
                        case NVMEDIA_IMAGE_CAPTURE_CSI_INTERFACE_TYPE_CSI_GH:
                        case NVMEDIA_IMAGE_CAPTURE_CSI_INTERFACE_TYPE_CSI_G:
                        case NVMEDIA_IMAGE_CAPTURE_CSI_INTERFACE_TYPE_CSI_H:
                            grpIndex = 3;
                            break;
                    }

                    LOG_INFO("Register with compositor for output:%u of sensor:%u\n", eOutput, uSensor);
                    auto outIndex = int(eOutput) - int(INvSIPLClient::ConsumerDesc::OutputType::ICP);
                    auto isRawOutput = (eOutput == INvSIPLClient::ConsumerDesc::OutputType::ICP) and isRawSensor;
                    status = upComposite->RegisterSource(grpIndex,
                                                         module.linkIndex,
                                                         outIndex,
                                                         isRawOutput,
                                                         uID,
                                                         &consumer[uSensor][(uint32_t)eOutput],
                                                         &consumerUpstream[uSensor][(uint32_t)eOutput],
                                                         &queue[uSensor][(uint32_t)eOutput]);
                    CHK_STATUS_AND_RETURN(status, "Composite register");
                }

                LOG_INFO("Consumer initialization for output:%u of sensor:%u\n", eOutput, uSensor);
                if (cmdline.bNvSci) {
                    status = upCons->Init(nullptr,
                                          (upComposite != nullptr) ? (CNvSIPLMasterNvSci *)upMaster.get() : nullptr,
                                          -1,
                                          uSensor,
                                          eOutput,
                                          cmdline.sPrefix,
                                          cmdline.uNumSkipFrames,
                                          cmdline.uNumWriteFrames);
                } else {
                    status = upCons->Init(upComposite.get(),
                                          nullptr,
                                          uID,
                                          uSensor,
                                          eOutput,
                                          cmdline.sPrefix,
                                          cmdline.uNumSkipFrames,
                                          cmdline.uNumWriteFrames);
                }
                if (status != NVSIPL_STATUS_OK) {
                    LOG_ERR("Failed to initialize consumer for output:%u of sensor:%u\n", eOutput, uSensor);
                    return -1;
                }

                if (cmdline.bShowMetadata) {
                    upCons->EnableMetadataLogging();
                }

                INvSIPLClient::ConsumerDesc client;
                client.eOutputType = eOutput;
                client.callback = upCons.get();
                pipelineCfg.clients.push_back(client);

                vupConsumers.push_back(move(upCons));
            } // output

            pipelineCfg.notifier = upNotificationHandler.get();

            NvSIPLDownscaleCropCfg &downscaleCropCfg = pipelineCfg.downscaleCropCfg;
            downscaleCropCfg.ispInputCropEnable = cmdline.bIspInputCropEnable;
            downscaleCropCfg.ispInputCrop.x0 = 0u;
            downscaleCropCfg.ispInputCrop.y0 = cmdline.uIspInputCropY;
            downscaleCropCfg.ispInputCrop.x1 = sensor.vcInfo.resolution.width;
            downscaleCropCfg.ispInputCrop.y1 = cmdline.uIspInputCropY + cmdline.uIspInputCropH;

            // Check if simulator mode.
            // If so, we need to set up a fileReader to feed frames to SIPL
            if (db.isSimulatorModeEnabled) {
                pipelineCfg.imageGroupWriter = fileReader.get();
            }

            status = upMaster->SetPipelineConfig(uSensor, pipelineCfg);
            CHK_STATUS_AND_RETURN(status, "Master SetPipelineConfig");
        } // module
    } // device block

    LOG_INFO("Initializing master interface\n");
    status = upMaster->Init();
    CHK_STATUS_AND_RETURN(status, "Master initialization");

    if (cmdline.bNvSci) {
        // for each sensor
        for (auto d = 0u; d != oPlatformCfg.numDeviceBlocks; d++) {
            auto db = oPlatformCfg.deviceBlockList[d];
            for (auto m = 0u; m != db.numCameraModules; m++) {
                auto module = db.cameraModuleInfoList[m];
                auto sensor = module.sensorInfo;
                uint32_t uSensor = sensor.id;

                // for each output
                for (INvSIPLClient::ConsumerDesc::OutputType eOutput : eOutputList) {
                    auto status = upMaster->RegisterSource(uSensor,
                                                           eOutput,
                                                           db.isSimulatorModeEnabled,
                                                           (upComposite != nullptr),
                                                           &consumer[uSensor][(uint32_t)eOutput],
                                                           consumerUpstream[uSensor][(uint32_t)eOutput],
                                                           &queue[uSensor][(uint32_t)eOutput]);
                    if (status != NVSIPL_STATUS_OK) {
                        LOG_ERR("Failed to register source for output:%u of sensor:%u\n", eOutput, uSensor);
                        return -1;
                    }
                } // output
                // ICP is needed even if not an output; register so that buffers can be allocated
                if (cmdline.bDisableRaw) {
                    status = upMaster->RegisterSource(uSensor,
                                                      INvSIPLClient::ConsumerDesc::OutputType::ICP,
                                                      db.isSimulatorModeEnabled,
                                                      false,
                                                      nullptr,
                                                      nullptr,
                                                      nullptr);
                    if (status != NVSIPL_STATUS_OK) {
                        LOG_ERR("Failed to register source for ICP output of sensor:%u\n", uSensor);
                        return -1;
                    }
                }
            } // module
        } // device block
    }

    if (cmdline.bAutoRecovery) {
        m_upAutoRecovery.reset(new CAutoRecovery(upMaster.get()));
        CHK_PTR_AND_RETURN(m_upAutoRecovery, "Auto recovery creation");
    }

    LOG_INFO("Starting compositor\n");
    if (upComposite != nullptr) {
        status = upComposite->Start();
        CHK_STATUS_AND_RETURN(status, "Compositor start");

        if (cmdline.bNvSci) {
            status = upMaster->SetupStreaming();
            if(status != NVSIPL_STATUS_OK) {
                LOG_ERR("Failed to set up producer streaming\n");
                return -1;
            }
        }
    }

    for (auto d = 0u; d != oPlatformCfg.numDeviceBlocks; d++ ) {
        auto db = oPlatformCfg.deviceBlockList[d];
        for (auto m = 0u; m != db.numCameraModules; m++) {
            auto module = db.cameraModuleInfoList[m];
            auto sensor = module.sensorInfo;
            uint32_t uSensor = sensor.id;

            if (cmdline.bShowEEPROM) {
                LOG_MSG("EEPROM of Sensor %d\n", uSensor);
                upMaster->ShowEEPROM(uSensor);
            }

            if (cmdline.bNvSci) {
                status = upMaster->AllocateBuffers(uSensor,
                                                   !cmdline.bDisableISP0,
                                                   !cmdline.bDisableISP1);
                if (status != NVSIPL_STATUS_OK) {
                    LOG_WARN("AllocateBuffers failed for sensor pipeline:%u\n", uSensor);
                }
            } else {
                status = upMaster->AllocateAndRegisterBuffers(uSensor,
                                                              !cmdline.bDisableISP0,
                                                              !cmdline.bDisableISP1);
                if (status != NVSIPL_STATUS_OK) {
                    LOG_WARN("AllocateAndRegisterBuffers failed for sensor pipeline:%u\n", uSensor);
                }
            }
        }
    }

    if ((upComposite != nullptr) && cmdline.bNvSci) {
        status = upMaster->SetupBuffers();
        if(status != NVSIPL_STATUS_OK) {
            LOG_ERR("Failed to set up producer buffers\n");
            return -1;
        }
    }

    if (cmdline.bNvSci) {
        for (auto d = 0u; d != oPlatformCfg.numDeviceBlocks; d++ ) {
            auto db = oPlatformCfg.deviceBlockList[d];
            for (auto m = 0u; m != db.numCameraModules; m++) {
                auto module = db.cameraModuleInfoList[m];
                auto sensor = module.sensorInfo;
                uint32_t uSensor = sensor.id;
                status = upMaster->RegisterBuffers(uSensor,
                                                   !cmdline.bDisableISP0,
                                                   !cmdline.bDisableISP1);
                if (status != NVSIPL_STATUS_OK) {
                    LOG_WARN("RegisterBuffers failed for sensor pipeline:%u\n", uSensor);
                }
            }
        }
    }

    std::unique_ptr<CAutoControlPlugin> upCustomPlugins[MAX_SENSORS_PER_PLATFORM] {nullptr};

    for (auto d = 0u; d != oPlatformCfg.numDeviceBlocks; d++ ) {
        auto db = oPlatformCfg.deviceBlockList[d];
        for (auto m = 0u; m != db.numCameraModules; m++) {
            auto module = db.cameraModuleInfoList[m];
            auto sensor = module.sensorInfo;
            uint32_t uSensor = sensor.id;

            if (!cmdline.bDisableISP0 || !cmdline.bDisableISP1) {
                std::vector<std::uint8_t> blob;
                status = LoadNITOFile(cmdline.sNitoFolderPath,
                                      module.name,
                                      blob);
                if (status != NVSIPL_STATUS_OK) {
                    LOG_ERR("Failed to load NITO file\n");
                    return -1;
                }

                if (cmdline.autoPlugin == NV_PLUGIN) {
                    status = upMaster->RegisterAutoControl(uSensor, NV_PLUGIN,
                                                           nullptr, blob);
                    if (status != NVSIPL_STATUS_OK) {
                        LOG_ERR("SetSiplControl(NV) failed for ISP output of sensor:%u\n", uSensor);
                        return -1;
                    }
                } else {
                    upCustomPlugins[uSensor].reset(new CAutoControlPlugin);
                    CHK_PTR_AND_RETURN(upCustomPlugins[uSensor], "AutoControl plugin creation");

                    status = upMaster->RegisterAutoControl(uSensor, CUSTOM_PLUGIN0,
                                                           upCustomPlugins[uSensor].get(), blob);
                    if (status != NVSIPL_STATUS_OK) {
                        LOG_ERR("SetAutoControl(CUST0) failed for ISP output of sensor:%u\n", uSensor);
                        return -1;
                    }
                }
            }
        }
    }

    LOG_INFO("Starting master\n");
    status = upMaster->Start();
    CHK_STATUS_AND_RETURN(status, "Master start");

    if (m_upAutoRecovery != nullptr) {
        LOG_INFO("Starting auto recovery\n");
        status = m_upAutoRecovery->Start();
        CHK_STATUS_AND_RETURN(status, "Auto recovery start");
    }

    // Spawn a background thread to accept user's runtime command
    std::thread([&]
    {
        pthread_setname_np(pthread_self(), "RuntimeMenu");

        while (!bQuit) {
            if (upComposite != nullptr) {
                cout << "Enter 'ld' to list display-able outputs.\n";
                cout << "Enter 'e' followed by 'ID' to enable display for specific camera group\n";
            }
            cout << "Enter 'les' followed by sensor ID to enable LED\n";
            cout << "Enter 'lds' followed by sensor ID to disable LED\n";
            cout << "Enter 'q' to quit the application\n";
            char line[256];
            cout << "-\n";
            cin.getline(line, 256);
            if (line[0] == 'q') {
                bQuit = true;
            } else if (upComposite != nullptr) {
                if (line[0] == 'l' and line[1] == 'd') {
                    upComposite->PrintDisplayableGroups();
                } else if (line[0] == 'e') {
                    auto uActiveDisplayable = atoi(&line[1]);
                    cout << "Enabled output" << uActiveDisplayable << endl;
                    upComposite->SetActiveGroup(uActiveDisplayable);
                }
            }

            if((line[0] == 'l') && (line[2] == 's')) {
                auto id = atoi(&line[3]);
                if ((id >= 0) && (id < 16)) {
                    if (line[1] == 'e') {
                        upMaster->ToggleLED(id, true);
                        cout << "Enable LED: " << id << endl;
                    } else if (line[1] == 'd') {
                        upMaster->ToggleLED(id, false);
                        cout << "Disable LED: " << id << endl;
                    }
                } else {
                    cout << "The sensor id " << id << " is out of the range. The valid sensor id is from 0 to 15" << endl;
                }
            }
        }
    }).detach();

    bool bFirstIteration = true;
    bool bValidFrameCount = false;
    uint64_t uFrameCountDelta = 0u;
    uint32_t uFrameDropPrintTimer = 0u;
    double firstLatencyMs = 0.0;
    double avgLatencyMs = 0.0;
    double rmsJitterMs = 0.0;
    // Wait for quit
    while (!bQuit) {
        // Wait for SECONDS_PER_ITERATION
        auto oStartTime = chrono::steady_clock::now();
        std::this_thread::sleep_for(std::chrono::seconds(SECONDS_PER_ITERATION));

        if (cmdline.uRunDurationSec != -1) {
            cmdline.uRunDurationSec -= SECONDS_PER_ITERATION;
            if (cmdline.uRunDurationSec < SECONDS_PER_ITERATION) {
                bQuit = true;
            }
        }

        // Check for any asynchronous fatal errors reported by pipeline threads in the library
        if (upNotificationHandler->IsPipelineInError()) {
            bQuit = true;
        }

        if (cmdline.bEnableProfiling) {
            uFrameDropPrintTimer += SECONDS_PER_ITERATION;
        }

        // Log FPS
        if (cmdline.bShowFPS || cmdline.bEnableProfiling) {
            auto uTimeElapsedMs = chrono::duration<double, std::milli> (chrono::steady_clock::now() - oStartTime).count();
            cout << "Output" << endl;
            for (auto &cons : vupConsumers) {
                cons->m_profData.profDataMut.lock();
                uFrameCountDelta = cons->m_profData.uFrameCount - cons->m_profData.uPrevFrameCount;
                cons->m_profData.uPrevFrameCount = cons->m_profData.uFrameCount;
                bValidFrameCount = uFrameCountDelta != 0u;
                if (cmdline.bEnableProfiling && bValidFrameCount) {
                    if (bFirstIteration) {
                        firstLatencyMs = cons->m_profData.uFirstLatencyUs / 1000.0;
                    }
                    avgLatencyMs = (cons->m_profData.uTotLatencyUs / 1000.0) / uFrameCountDelta;
                    rmsJitterMs = sqrt(cons->m_profData.uTotSqJitterUs / uFrameCountDelta) / 1000.0;
                    cons->m_profData.uTotLatencyUs = 0u;
                    cons->m_profData.uTotSqJitterUs = 0u;
                }
                cons->m_profData.profDataMut.unlock();
                auto fps = uFrameCountDelta / (uTimeElapsedMs / 1000.0);
                string consName = "Sensor" + to_string(cons->m_uSensor) + "_Out"
                                  + to_string(int(cons->m_outputType));
                cout << consName << "\tFrame rate (fps):\t" << fps << endl;
                if (cmdline.bEnableProfiling) {
                    if (bFirstIteration) {
                        cout << consName << "\tFirst latency (ms):\t";
                        if (bValidFrameCount) {
                            cout << firstLatencyMs << endl;
                        } else {
                            cout << "N/A" << endl;
                        }
                    }
                    cout << consName << "\tAverage latency (ms):\t";
                    if (bValidFrameCount) {
                        cout << avgLatencyMs << endl;
                    } else {
                        cout << "N/A" << endl;
                    }
                    cout << consName << "\tRMS jitter (ms):\t";
                    if (bValidFrameCount) {
                        cout << rmsJitterMs << endl;
                    } else {
                        cout << "N/A" << endl;
                    }
                    // Only print number of frame drops each hour
                    if (uFrameDropPrintTimer >= SECONDS_PER_HOUR) {
                        auto numFrameDrops = upNotificationHandler->GetNumFrameDrops(cons->m_uSensor);
                        cout << consName << "\tNumber of frame drops:\t" << numFrameDrops << endl;
                    }
                    cout << endl;
                }
            }
            if (cmdline.bEnableProfiling) {
                if (bFirstIteration) {
                    bFirstIteration = false;
                }
                if (uFrameDropPrintTimer >= SECONDS_PER_HOUR) {
                    uFrameDropPrintTimer = 0u;
                    upNotificationHandler->ResetFrameDropCounter();
                }
            }
            cout << endl;
        }
    }

    if (m_upAutoRecovery != nullptr) {
        LOG_INFO("Stopping auto recovery\n");
        m_upAutoRecovery->Stop();
    }

    if (upMaster != nullptr) {
        LOG_INFO("Stopping master\n");
        status = upMaster->Stop();
        if (status != NVSIPL_STATUS_OK) {
            LOG_ERR("Failed to Stop master %u\n", status);
        }
    }

    if (upComposite != nullptr) {
        LOG_INFO("Stopping compositor\n");
        upComposite->Stop();
    }

    if (fileReader != nullptr) {
        fileReader->Deinit();
    }

    if (upComposite != nullptr) {
        LOG_INFO("De-initializing compositor\n");
        upComposite->Deinit();
    }

    if (upMaster != nullptr) {
        LOG_INFO("De-initilaizing master\n");
        upMaster->Deinit();
    }

    for (auto &cons : vupConsumers) {
        string consumerName = "Sensor" + to_string(cons->m_uSensor) + "_Out"
                              + to_string(int(cons->m_outputType));
        LOG_INFO("De-initializing consumer %s\n", consumerName.c_str());
        cons->Deinit();
    }

    // Check for any asynchronous fatal errors reported by pipeline threads in the library
    if (upNotificationHandler->IsPipelineInError()) {
        LOG_ERR("Pipeline failure\n");
        return -1;
    }

    LOG_MSG("SUCCESS\n");
    return 0;
}
