Hello community,
I am trying to get a trace buffer setup and working during an event of crash on RPU R5F cores. I am using Xilinx Zynq MP board with both Cortex-R5F on split mode. I build a minimal RPU firmware application where I try to force a crash and get the trace details from trace buffer.
What I have done so far (along with Xilinx support personnel help) 1. Built the libmetal latest version 1.7.0 from https://github.com/OpenAMP/libmetal 2. Built the openamp latest version 1.7.0 from https://github.com/OpenAMP/open-amp 3. Built the RPU firmware app with both these libraries, I had to build and link both to use the metal debug feature.
In the app, I create a resource table for the trace buffer as follows: struct remote_resource_table __resource resources = { .version = 1, .num = NUM_TABLE_ENTRIES, .reserved = {0, 0}, .offset[0] = offsetof(struct remote_resource_table, rpmsg_vdev), .offset[1] = offsetof(struct remote_resource_table, rsc_trace), /* Virtio device entry */ .rpmsg_vdev = { .type = RSC_VDEV, .id = VIRTIO_ID_RPMSG_, .notifyid = 31, .dfeatures = RPMSG_VDEV_DFEATURES, .gfeatures = 0, .config_len = 0, .status = 0, .num_of_vrings = NUM_VRINGS, .reserved = {0, 0}, }, /* Vring rsc entry - part of vdev rsc entry */ .rpmsg_vring0 = {RING_TX, VRING_ALIGN, VRING_SIZE, 1, 0}, .rpmsg_vring1 = {RING_RX, VRING_ALIGN, VRING_SIZE, 2, 0}, /* trace buffer for logs, accessible via debugfs */ .rsc_trace = { .type = RSC_TRACE, .da = (unsigned int)rsc_trace_buf, .len = sizeof(rsc_trace_buf), .reserved = 0, .name = "r5_trace", }, };
and in linker file, added the section in ddr as below:
/* Define Memories in the system */
MEMORY { psu_ocm_ram_0_MEM_0 : ORIGIN = 0xFFFC0000, LENGTH = 0x40000 psu_r5_0_atcm_MEM_0 : ORIGIN = 0x0, LENGTH = 0x10000 psu_r5_0_btcm_MEM_0 : ORIGIN = 0x20000, LENGTH = 0x10000 psu_r5_ddr_0_MEM_0 : ORIGIN = 0x3ed00000, LENGTH = 0x40000 }
/* Specify the default entry point to the program */
ENTRY(_boot)
/* Define the sections, and where they are mapped in memory */
SECTIONS {
.resource_table 0x3ed20000 : { . = ALIGN(4); *(.resource_table) } > psu_r5_ddr_0_MEM_0
Now when I try to create a core dump file by starting and running the RPU app from userspace, after I disable recovery and enable coredump, I don’t see any trace info about the coredump in the trace file other than print statements I added. See below. echo “disabled” > /sys/kernel/debug/remoteproc/remoteproc0/recovery echo “enabled” > /sys/kernel/debug/remoteproc/remoteproc0/enabled
root@zcu102-zynqmp:/sys/kernel/debug/remoteproc/remoteproc0# ls -altr total 0 drwxr-xr-x 4 root root 0 Jan 1 1970 .. -r-------- 1 root root 0 Jan 14 18:05 resource_table -rw------- 1 root root 0 Jan 14 18:05 recovery -r-------- 1 root root 0 Jan 14 18:05 name --w------- 1 root root 0 Jan 14 18:05 crash -rw------- 1 root root 0 Jan 14 18:05 coredump -r-------- 1 root root 0 Jan 14 18:05 carveout_memories drwxr-xr-x 2 root root 0 Jan 14 18:05 . -r-------- 1 root root 0 Jan 14 18:10 trace0 root@zcu102-zynqmp:/sys/kernel/debug/remoteproc/remoteproc0# cat trace0 0 L7 registered generic bus 1 L6 MCS-sp: c_buf,c_len = 0x3ed3a7ec,40962 L6 MCS-sp: platform_create_proc()3 L6 ML:platform_create_proc() 4 L6 MCS-sp: rsc_table, rsc_size = 0x3ed20000, 0x1005 L6 MCS-sp: metal_device_open(generic, poll_dev, 0x23b4) 6 L6 MCS-sp: poll{name,bus,chn_mask} = poll_dev,generic,0x10000007 L6 MCS-sp: lpa,lda= 0x3ed20000,0xffffffff 8 L6 MCS-sp: mem= 0x34d8 9 L6 MCS-sp: tmpio= 0x3518 10 L6 MCS-sp: lpa,lda= 0x3ed40000,0xffffffff 11 L6 MCS-sp: mem= 0x3558 12 L6 MCS-sp: tmpio= 0x3598 13 L6 MCS-sp: Initialize remoteproc successfully.14 L6 MCS-sp: creating remoteproc virtio rproc 0x3ed3a7b4 15 L6 MCS-sp: initializing rpmsg shared buffer pool 16 L6 MCS-sp: initializing rpmsg vdev 17 L6 MCS-sp: trying to coredump root@zcu102-zynqmp:/sys/kernel/debug/remoteproc/remoteproc0#
I also tried with “recovery” state enabled too with no luck. Any pointers on what I may be missing here? Hopefully I can get some help from the openamp community, appreciate the help! Copied file rpu.c is the main file where init_platform is called to init the platform and init_system() is called, and platform.c has these functions defined along with the resource table below.
Thanks Siva
rpu.c file: ----------------------------------------------------------------------- extern struct rpmsg_device * platform_create_rpmsg_vdev(void *platform, uint32_t vdev_index, uint32_t role, void (*rst_cb)(struct virtio_device *vdev), rpmsg_ns_bind_cb ns_bind_cb);
int main(void) { unsigned long sweep_count = 0;
void *platform; init_platform(&platform);
struct rpmsg_device *rpdev; rpdev = platform_create_rpmsg_vdev(platform, 0, VIRTIO_DEV_DEVICE, NULL, NULL); if (!rpdev) { metal_log(METAL_LOG_INFO,"MCS-sp: Failed to create rpmsg virtio device.\r\n"); metal_log(METAL_LOG_INFO,"MCS-sp: RPU reboot is required to recover\r\n"); } while (1) { usleep(10); if(sweep_count++ >= 0x400000) { printf("MCS-sp: trying to coredump"); metal_log(METAL_LOG_INFO,"MCS-sp: trying to coredump\r\n"); sleep (1); metal_log(METAL_LOG_INFO,"MCS-sp: coredump here %d", (1/0)); printf("MCS-sp: coredump here %d", (1/0)); } } return 0; }
platform.c file snippet ---------------------------------------------------------------------------------------------- static struct remoteproc * platform_create_proc(uint32_t proc_index, uint32_t rsc_index) { void *rsc_table; uint32_t rsc_size; int32_t ret; metal_phys_addr_t pa;
(void) proc_index; rsc_table = get_resource_table(rsc_index, &rsc_size); metal_log(METAL_LOG_INFO,"MCS-sp: rsc_table, rsc_size = %#x, %#x", rsc_table, rsc_size);
/* Register IPI device */ if (metal_register_generic_device(&kick_device)) return NULL;
/* Initialize remoteproc instance */ if (!remoteproc_init(&rproc_inst, &zynqmp_r5_a53_proc_ops, &rproc_priv)) return NULL;
metal_log(METAL_LOG_INFO,"MCS-sp: poll{name,bus,chn_mask} = %s,%s,%#x", rproc_priv.kick_dev_name, rproc_priv.kick_dev_bus_name, IPI_CHN_BITMASK); /* * Mmap shared memories * Or shall we constraint that they will be set as carved out * in the resource table? */ /* mmap resource table */ pa = (metal_phys_addr_t)rsc_table; (void *)remoteproc_mmap(&rproc_inst, &pa, NULL, rsc_size, NORM_NSHARED_NCACHE|PRIV_RW_USER_RW, &rproc_inst.rsc_io); /* mmap shared memory */ pa = SHARED_MEM_PA; (void *)remoteproc_mmap(&rproc_inst, &pa, NULL, SHARED_MEM_SIZE, NORM_NSHARED_NCACHE|PRIV_RW_USER_RW, NULL);
/* parse resource table to remoteproc */ ret = remoteproc_set_rsc_table(&rproc_inst, rsc_table, rsc_size); if (ret != 0) { metal_log(METAL_LOG_INFO,"MCS-sp: Failed to initialize remoteproc"); remoteproc_remove(&rproc_inst); return NULL; } metal_log(METAL_LOG_INFO,"MCS-sp: Initialize remoteproc successfully.");
return &rproc_inst; }
int32_t init_system(void); struct rpmsg_device * platform_create_rpmsg_vdev(void *platform, uint32_t vdev_index, uint32_t role, void (*rst_cb)(struct virtio_device *vdev), rpmsg_ns_bind_cb ns_bind_cb); int init_platform(void **platform) { /* metal_log setup is in init_system */ if (!platform) { metal_log(METAL_LOG_INFO,"MCS-sp: Failed to initialize platform," "NULL pointer to store platform data."); return -EINVAL; } init_system(); unsigned long proc_id = 0; unsigned long rsc_id = 0; metal_log(METAL_LOG_INFO, "MCS-sp: platform_create_proc()"); ML_INFO("ML:platform_create_proc()\r\n"); struct remoteproc *rproc; rproc = platform_create_proc(proc_id, rsc_id); if (!rproc) { metal_log(METAL_LOG_INFO,"MCS-sp: Failed to create remoteproc device."); return -EINVAL; } *platform = rproc; return 0; }
void cleanup_platform() { disable_caches(); }
struct rpmsg_device * platform_create_rpmsg_vdev(void *platform, uint32_t vdev_index, uint32_t role, void (*rst_cb)(struct virtio_device *vdev), rpmsg_ns_bind_cb ns_bind_cb) { struct remoteproc *rproc = platform; struct rpmsg_virtio_device *rpmsg_vdev; struct virtio_device *vdev; void *shbuf; struct metal_io_region *shbuf_io; int32_t ret; struct rpmsg_device *ret_rpmsg_dev=NULL; rpmsg_vdev = metal_allocate_memory(sizeof(*rpmsg_vdev)); if (!rpmsg_vdev) return NULL; shbuf_io = remoteproc_get_io_with_pa(rproc, SHARED_MEM_PA); if (!shbuf_io){ goto err1; } shbuf = metal_io_phys_to_virt(shbuf_io, SHARED_MEM_PA + SHARED_BUF_OFFSET);
metal_log(METAL_LOG_INFO,"MCS-sp: creating remoteproc virtio rproc %p\r\n", rproc); /* TODO: can we have a wrapper for the following two functions? */ vdev = remoteproc_create_virtio(rproc, vdev_index, role, rst_cb); if (!vdev) { metal_log(METAL_LOG_INFO,"MCS-sp: failed remoteproc_create_virtio\r\n"); goto err1; }
metal_log(METAL_LOG_INFO,"MCS-sp: initializing rpmsg shared buffer pool\r\n"); /* Only RPMsg virtio master needs to initialize the shared buffers pool */ rpmsg_virtio_init_shm_pool(&shpool, shbuf, (SHARED_MEM_SIZE - SHARED_BUF_OFFSET));
metal_log(METAL_LOG_INFO,"MCS-sp: initializing rpmsg vdev\r\n"); /* RPMsg virtio device can set shared buffers pool argument to NULL */ ret = rpmsg_init_vdev(rpmsg_vdev, vdev, ns_bind_cb, shbuf_io, &shpool); if (ret != 0) { metal_log(METAL_LOG_INFO,"MCS-sp: failed rpmsg_init_vdev\r\n"); goto err2; }
ret_rpmsg_dev = rpmsg_virtio_get_rpmsg_device(rpmsg_vdev);
if (rpmsg_vdev != NULL){ metal_free_memory(rpmsg_vdev); }
return ret_rpmsg_dev; err2: remoteproc_remove_virtio(rproc, vdev); err1: metal_free_memory(rpmsg_vdev); return NULL; }
static void rsc_trace_putchar(char c) { if (circ.c_pos >= circ.c_len) circ.c_pos = 0; circ.c_buf[circ.c_pos++] = c; }
static void rsc_trace_logger(enum metal_log_level level, const char *format, ...) { char msg[128]; char *p; int32_t len; va_list args;
/* prefix "cnt L6 ": record count and log level */ len = sprintf(msg, "%lu L%u ", circ.c_cnt, level); if (len < 0 || len >= sizeof(msg)) len = 0; circ.c_cnt++;
va_start(args, format); vsnprintf(msg + len, sizeof(msg) - len, format, args); va_end(args);
/* copy at most sizeof(msg) to the circular buffer */ for (len = 0, p = msg; *p && len < sizeof(msg); ++len, ++p) rsc_trace_putchar(*p); /* Remove this xil_printf to stop printing to console */ xil_printf("%s", msg); }
char *get_rsc_trace_info(uint32_t *len) { *len = sizeof(rsc_trace_buf); return rsc_trace_buf; } void *get_resource_table (uint32_t rsc_id, uint32_t *len) { (void) rsc_id; *len = sizeof(resources); return &resources; }
int32_t init_system(void) { int32_t ret; struct metal_init_params metal_param = METAL_INIT_DEFAULTS;
circ.c_buf = get_rsc_trace_info(&circ.c_len); if (circ.c_buf && circ.c_len){ metal_param.log_handler = &rsc_trace_logger; metal_param.log_level = METAL_LOG_DEBUG; circ.c_pos = circ.c_cnt = 0; };
/* Low level abstraction layer for openamp initialization */ metal_init(&metal_param);
/* configure the global interrupt controller */ app_gic_initialize();
/* Initialize metal Xilinx IRQ controller */ ret = metal_xlnx_irq_init(); if (ret != 0) { metal_log(METAL_LOG_INFO,"MCS-sp: metal_xlnx_irq_init failed."); }
metal_log(METAL_LOG_INFO,"MCS-sp: c_buf,c_len = %p,%u", circ.c_buf, circ.c_len); return ret; }