Issue in configuring USBX in non-secure world on Trustzone board STM32U585

I am facing issue in running the thread USBX application in non-secure world on STM32U585. Below is the problem description and solution I tried. 

Problem Description :

               I am trying to run USBX application in non-secure world and SAES-GCM crypto application in secure world. The crypto application is working fine meaning able to encrypt and decrypt the data.

But as it switches to non-secure world, after the Initialization, application stuck in I2C error occurred API, when trying to initialize CAD (USPD_CAD_Init “API name”). As this API is library API, I am not able to further debug it.

Solution tried :

1. We set SystemClock  to 16MHZ with PLL source = HSI and systemclock is configured only from secure world
   1. Do we need to configure SystemClock in both the world or it is done once only?

1. If building standalone USBX application after disabling the trustzone, it is working fine.

May be I am setting some of the configuration wrong, so can anyone guide us on this.

Below is the Clock Configuration I am configuring in secure world(assuming it will help to configure clock of both world)

/***********************Clock Configuration Begins********************************************

RCC_ClkInitTypeDef RCC_ClkInitStruct = {0};
RCC_OscInitTypeDef RCC_OscInitStruct = {0};
RCC_CRSInitTypeDef RCC_CRSInitStruct = {0};

HAL_PWREx_ControlVoltageScaling(PWR_REGULATOR_VOLTAGE_SCALE1)

/* MSI Oscillator enabled at reset (4Mhz), activate PLL with MSI as source */
RCC_OscInitStruct.OscillatorType = RCC_OSCILLATORTYPE_HSI48|RCC_OSCILLATORTYPE_HSI
|RCC_OSCILLATORTYPE_SHSI;
RCC_OscInitStruct.HSIState = RCC_HSI_ON;
RCC_OscInitStruct.HSI48State = RCC_HSI48_ON;
RCC_OscInitStruct.HSICalibrationValue = RCC_HSICALIBRATION_DEFAULT;
// RCC_OscInitStruct.MSIState = RCC_MSI_ON;
//RCC_OscInitStruct.MSICalibrationValue = RCC_MSICALIBRATION_DEFAULT;
// RCC_OscInitStruct.MSIClockRange = RCC_MSIRANGE_4;
RCC_OscInitStruct.SHSIState = RCC_SHSI_ON;
RCC_OscInitStruct.PLL.PLLState = RCC_PLL_ON;
// RCC_OscInitStruct.PLL.PLLSource = RCC_PLLSOURCE_MSI;
RCC_OscInitStruct.PLL.PLLSource = RCC_PLLSOURCE_HSI;
RCC_OscInitStruct.PLL.PLLMBOOST = RCC_PLLMBOOST_DIV1;
RCC_OscInitStruct.PLL.PLLM = 1;
RCC_OscInitStruct.PLL.PLLN = 10;// 80 from crypto
RCC_OscInitStruct.PLL.PLLP = 2;
RCC_OscInitStruct.PLL.PLLQ = 2;
RCC_OscInitStruct.PLL.PLLR = 1; //2 for crypto
RCC_OscInitStruct.PLL.PLLRGE = RCC_PLLVCIRANGE_1; //for crypto RCC_PLLVCIRANGE_0
RCC_OscInitStruct.PLL.PLLFRACN= 0;
if (HAL_RCC_OscConfig(&RCC_OscInitStruct) != HAL_OK)
{
/* Initialization Error */
while(1);
}

/* Select PLL as system clock source and configure bus clocks dividers */
RCC_ClkInitStruct.ClockType = (RCC_CLOCKTYPE_SYSCLK | RCC_CLOCKTYPE_HCLK | RCC_CLOCKTYPE_PCLK1 | \
RCC_CLOCKTYPE_PCLK2 | RCC_CLOCKTYPE_PCLK3);
RCC_ClkInitStruct.SYSCLKSource = RCC_SYSCLKSOURCE_PLLCLK;
RCC_ClkInitStruct.AHBCLKDivider = RCC_SYSCLK_DIV1;
RCC_ClkInitStruct.APB1CLKDivider = RCC_HCLK_DIV1;
RCC_ClkInitStruct.APB2CLKDivider = RCC_HCLK_DIV1;
RCC_ClkInitStruct.APB3CLKDivider = RCC_HCLK_DIV1;

if(HAL_RCC_ClockConfig(&RCC_ClkInitStruct, FLASH_LATENCY_4) != HAL_OK)
{
Error_Handler();
/* Initialization Error */
}

__HAL_RCC_CRS_CLK_ENABLE();

/** Configures CRS
*/
RCC_CRSInitStruct.Prescaler = RCC_CRS_SYNC_DIV1;
RCC_CRSInitStruct.Source = RCC_CRS_SYNC_SOURCE_USB;
RCC_CRSInitStruct.Polarity = RCC_CRS_SYNC_POLARITY_RISING;
RCC_CRSInitStruct.ReloadValue = __HAL_RCC_CRS_RELOADVALUE_CALCULATE(48000000,1000);
RCC_CRSInitStruct.ErrorLimitValue = 34;
RCC_CRSInitStruct.HSI48CalibrationValue = 32;

HAL_RCCEx_CRSConfig(&RCC_CRSInitStruct);

/********************************Clock Configuration Ends*****************************************************/