Skip to content

Commit ec987ee

Browse files
edgargabrieledgargabriel
authored
Merge pull request #12949 from edgargabriel/topic/smsc-accelerator
smsc/accelerator: add new smsc component
2 parents 99bec5a + 4a117d5 commit ec987ee

11 files changed

+565
-15
lines changed

‎opal/mca/accelerator/rocm/accelerator_rocm_module.c

+7-1
Original file line numberDiff line numberDiff line change
@@ -390,6 +390,12 @@ static int mca_accelerator_rocm_memcpy(int dest_dev_id, int src_dev_id, void *de
390390
"error during synchronous copy\n");
391391
return OPAL_ERROR;
392392
}
393+
err = hipStreamSynchronize(0);
394+
if (hipSuccess != err ) {
395+
opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
396+
"error synchronizing default stream after hipMemcpy\n");
397+
return OPAL_ERROR;
398+
}
393399
}
394400

395401
return OPAL_SUCCESS;
@@ -939,4 +945,4 @@ static int mca_accelerator_rocm_get_mem_bw(int device, float *bw)
939945

940946
*bw = opal_accelerator_rocm_mem_bw[device];
941947
return OPAL_SUCCESS;
942-
}
948+
}

‎opal/mca/btl/sm/btl_sm_component.c

+4
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,10 @@ mca_btl_sm_component_init(int *num_btls, bool enable_progress_threads, bool enab
340340
mca_btl_sm.super.btl_get = mca_btl_sm_get;
341341
mca_btl_sm.super.btl_put = mca_btl_sm_put;
342342

343+
if (mca_smsc_base_has_feature(MCA_SMSC_FEATURE_ACCELERATOR)) {
344+
mca_btl_sm.super.btl_flags |= MCA_BTL_FLAGS_ACCELERATOR_GET;
345+
}
346+
343347
mca_btl_sm.super.btl_bandwidth = 40000; /* Mbs */
344348

345349
if (mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)) {

‎opal/mca/smsc/accelerator/Makefile.am

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#
2+
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3+
# University Research and Technology
4+
# Corporation. All rights reserved.
5+
# Copyright (c) 2004-2009 The University of Tennessee and The University
6+
# of Tennessee Research Foundation. All rights
7+
# reserved.
8+
# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
9+
# University of Stuttgart. All rights reserved.
10+
# Copyright (c) 2004-2005 The Regents of the University of California.
11+
# All rights reserved.
12+
# Copyright (c) 2009-2014 Cisco Systems, Inc. All rights reserved.
13+
# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
14+
# reserved.
15+
# Copyright (c) 2017 IBM Corporation. All rights reserved.
16+
# Copyright (c) 2020-2021 Google, LLC. All rights reserved.
17+
# Copyright (c) 2024 Advanced Micro Devices, Inc. All Rights reserved.
18+
# $COPYRIGHT$
19+
#
20+
# Additional copyrights may follow
21+
#
22+
# $HEADER$
23+
#
24+
25+
EXTRA_DIST = post_configure.sh
26+
27+
AM_CPPFLAGS = $(smsc_accelerator_CPPFLAGS)
28+
29+
30+
libmca_smsc_accelerator_la_sources = \
31+
smsc_accelerator_component.c \
32+
smsc_accelerator_module.c \
33+
smsc_accelerator_internal.h \
34+
smsc_accelerator.h
35+
36+
# Make the output library in this directory, and name it either
37+
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
38+
# (for static builds).
39+
40+
if MCA_BUILD_opal_smsc_accelerator_DSO
41+
component_noinst =
42+
component_install = mca_smsc_accelerator.la
43+
else
44+
component_noinst = libmca_smsc_accelerator.la
45+
component_install =
46+
endif
47+
48+
mcacomponentdir = $(opallibdir)
49+
mcacomponent_LTLIBRARIES = $(component_install)
50+
mca_smsc_accelerator_la_SOURCES = $(libmca_smsc_accelerator_la_sources)
51+
mca_smsc_accelerator_la_LDFLAGS = -module -avoid-version $(smsc_accelerator_LDFLAGS)
52+
mca_smsc_accelerator_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
53+
$(smsc_accelerator_LIBS)
54+
55+
noinst_LTLIBRARIES = $(component_noinst)
56+
libmca_smsc_accelerator_la_SOURCES = $(libmca_smsc_accelerator_la_sources)
57+
libmca_smsc_accelerator_la_LIBADD = $(smsc_accelerator_LIBS)
58+
libmca_smsc_accelerator_la_LDFLAGS = -module -avoid-version $(smsc_accelerator_LDFLAGS)

‎opal/mca/smsc/accelerator/post_configure.sh

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
DIRECT_CALL_HEADER="opal/mca/smsc/accelerator/smsc_accelerator.h"

‎opal/mca/smsc/accelerator/smsc_accelerator.h

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2024 Advanced Micro Devices, Inc. All Rights reserved.
4+
* $COPYRIGHT$
5+
*
6+
* Additional copyrights may follow
7+
*
8+
* $HEADER$
9+
*/
10+
11+
#ifndef OPAL_MCA_SMSC_ACCELERATOR_H
12+
#define OPAL_MCA_SMSC_ACCELERATOR_H
13+
14+
#include "opal_config.h"
15+
16+
#include "opal/mca/smsc/smsc.h"
17+
18+
mca_smsc_endpoint_t *mca_smsc_accelerator_get_endpoint(opal_proc_t *peer_proc);
19+
void mca_smsc_accelerator_return_endpoint(mca_smsc_endpoint_t *endpoint);
20+
21+
int mca_smsc_accelerator_copy_to(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
22+
size_t size, void *reg_handle);
23+
24+
int mca_smsc_accelerator_copy_from(mca_smsc_endpoint_t *endpoint, void *local_address,
25+
void *remote_address, size_t size, void *reg_handle);
26+
27+
void *mca_smsc_accelerator_map_peer_region(mca_smsc_endpoint_t *endpoint, uint64_t flags,
28+
void *remote_ptr, size_t size, void **local_ptr);
29+
void mca_smsc_accelerator_unmap_peer_region(void *ctx);
30+
31+
void *mca_smsc_accelerator_register_region(void *local_address, size_t size);
32+
void mca_smsc_accelerator_deregister_region(void *reg_data);
33+
34+
#endif /* OPAL_MCA_SMSC_ACCELERATOR__H */

‎opal/mca/smsc/accelerator/smsc_accelerator_component.c

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2024 Advanced Micro Devices, Inc. All Rights reserved.
4+
* $COPYRIGHT$
5+
*
6+
* Additional copyrights may follow
7+
*
8+
* $HEADER$
9+
*/
10+
#include "opal_config.h"
11+
12+
#include "opal/mca/smsc/base/base.h"
13+
#include "opal/mca/smsc/accelerator/smsc_accelerator_internal.h"
14+
#include "opal/mca/accelerator/accelerator.h"
15+
#include "opal/mca/accelerator/base/base.h"
16+
17+
static int mca_smsc_accelerator_component_register(void);
18+
static int mca_smsc_accelerator_component_open(void);
19+
static int mca_smsc_accelerator_component_close(void);
20+
static int mca_smsc_accelerator_component_query(void);
21+
static mca_smsc_module_t *mca_smsc_accelerator_component_enable(void);
22+
23+
#define MCA_SMSC_ACCELERATOR_DEFAULT_PRIORITY 0
24+
static const int mca_smsc_accelerator_default_priority = MCA_SMSC_ACCELERATOR_DEFAULT_PRIORITY;
25+
26+
mca_smsc_accelerator_component_t mca_smsc_accelerator_component = {
27+
.super = {
28+
.smsc_version = {
29+
MCA_SMSC_DEFAULT_VERSION("accelerator"),
30+
.mca_open_component = mca_smsc_accelerator_component_open,
31+
.mca_close_component = mca_smsc_accelerator_component_close,
32+
.mca_register_component_params = mca_smsc_accelerator_component_register,
33+
},
34+
.priority = MCA_SMSC_ACCELERATOR_DEFAULT_PRIORITY,
35+
.query = mca_smsc_accelerator_component_query,
36+
.enable = mca_smsc_accelerator_component_enable,
37+
},
38+
};
39+
40+
static int mca_smsc_accelerator_component_register(void)
41+
{
42+
mca_smsc_base_register_default_params(&mca_smsc_accelerator_component.super,
43+
mca_smsc_accelerator_default_priority);
44+
return OPAL_SUCCESS;
45+
}
46+
47+
static int mca_smsc_accelerator_component_open(void)
48+
{
49+
return OPAL_SUCCESS;
50+
}
51+
52+
static int mca_smsc_accelerator_component_close(void)
53+
{
54+
if (mca_smsc_accelerator_module.rcache) {
55+
(void) mca_rcache_base_module_destroy(mca_smsc_accelerator_module.rcache);
56+
mca_smsc_accelerator_module.rcache = NULL;
57+
}
58+
59+
return OPAL_SUCCESS;
60+
}
61+
62+
static int mca_smsc_accelerator_component_query(void)
63+
{
64+
if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name,
65+
"null")) {
66+
opal_output_verbose(10, opal_smsc_base_framework.framework_output,
67+
"smsc:accelerator:component_query: accelerator component is null: disqualifying myself");
68+
return OPAL_ERROR;
69+
}
70+
71+
if (!opal_accelerator.is_ipc_enabled()) {
72+
opal_output_verbose(10, opal_smsc_base_framework.framework_output,
73+
"smsc:accelerator:component_query: accelerator component does not have support for IPC operations: disqualifying myself");
74+
return OPAL_ERROR;
75+
}
76+
77+
if (1 > mca_smsc_accelerator_component.super.priority) {
78+
return OPAL_ERROR;
79+
}
80+
81+
return OPAL_SUCCESS;
82+
}
83+
84+
static mca_smsc_module_t *mca_smsc_accelerator_component_enable(void)
85+
{
86+
if (0 > mca_smsc_accelerator_component.super.priority) {
87+
return NULL;
88+
}
89+
90+
mca_rcache_base_resources_t rcache_resources;
91+
mca_smsc_accelerator_module.rcache = mca_rcache_base_module_create("gpusm", NULL, &rcache_resources);
92+
if (NULL == mca_smsc_accelerator_module.rcache) {
93+
return NULL;
94+
}
95+
96+
/* Not set. Will initialize later */
97+
mca_smsc_accelerator_module.device_id = MCA_ACCELERATOR_NO_DEVICE_ID;
98+
99+
mca_smsc_accelerator_module.prev_smsc = mca_smsc;
100+
if ((NULL != mca_smsc_accelerator_module.prev_smsc) &&
101+
(mca_smsc_accelerator_module.prev_smsc->features & MCA_SMSC_FEATURE_CAN_MAP)) {
102+
mca_smsc_accelerator_module.super.features |= MCA_SMSC_FEATURE_CAN_MAP;
103+
}
104+
105+
return &mca_smsc_accelerator_module.super;
106+
}

‎opal/mca/smsc/accelerator/smsc_accelerator_internal.h

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2021 Google, Inc. All rights reserved.
4+
* Copyright (c) 2024 Advanced Micro Devices, Inc. All Rights reserved.
5+
* $COPYRIGHT$
6+
*
7+
* Additional copyrights may follow
8+
*
9+
* $HEADER$
10+
*/
11+
12+
#ifndef OPAL_MCA_SMSC_ACCELERATOR_INTERNAL_H
13+
#define OPAL_MCA_SMSC_ACCELERATOR_INTERNAL_H
14+
15+
#include "opal_config.h"
16+
17+
#include "opal/mca/rcache/base/base.h"
18+
#include "opal/mca/rcache/rcache.h"
19+
#include "opal/mca/smsc/accelerator/smsc_accelerator.h"
20+
#include "opal/mca/accelerator/accelerator.h"
21+
#include "opal/include/opal/opal_gpu.h"
22+
23+
#define SMSC_ACCELERATOR_HANDLE_SIZE IPC_MAX_HANDLE_SIZE
24+
struct mca_smsc_accelerator_registration_data_t {
25+
uint64_t base_addr;
26+
union {
27+
uint8_t accelerator[SMSC_ACCELERATOR_HANDLE_SIZE];
28+
void* host;
29+
} handle;
30+
};
31+
typedef struct mca_smsc_accelerator_registration_data_t mca_smsc_accelerator_registration_data_t;
32+
33+
struct mca_smsc_accelerator_registration_handle_t {
34+
mca_opal_gpu_reg_t *gpu_reg; // contains mca_rcache_base_registration_t base
35+
mca_smsc_accelerator_registration_data_t data;
36+
};
37+
typedef struct mca_smsc_accelerator_registration_handle_t mca_smsc_accelerator_registration_handle_t;
38+
OBJ_CLASS_DECLARATION(mca_smsc_accelerator_registration_handle_t);
39+
40+
#define MCA_SMSC_ACCELERATOR_REG_DATA_TO_HANDLE(data_ptr) \
41+
((mca_smsc_accelerator_registration_handle_t *) ((uintptr_t) data_ptr \
42+
- offsetof(mca_smsc_accelerator_registration_handle_t, \
43+
data)))
44+
45+
struct mca_smsc_accelerator_endpoint_t {
46+
mca_smsc_endpoint_t super;
47+
mca_smsc_endpoint_t *prev_endpoint;
48+
mca_rcache_base_module_t *rcache;
49+
};
50+
typedef struct mca_smsc_accelerator_endpoint_t mca_smsc_accelerator_endpoint_t;
51+
OBJ_CLASS_DECLARATION(mca_smsc_accelerator_endpoint_t);
52+
53+
struct mca_smsc_accelerator_component_t {
54+
mca_smsc_component_t super;
55+
};
56+
typedef struct mca_smsc_accelerator_component_t mca_smsc_accelerator_component_t;
57+
58+
struct mca_smsc_accelerator_module_t {
59+
mca_smsc_module_t super;
60+
mca_smsc_module_t *prev_smsc;
61+
mca_rcache_base_module_t *rcache;
62+
int device_id;
63+
};
64+
typedef struct mca_smsc_accelerator_module_t mca_smsc_accelerator_module_t;
65+
66+
extern mca_smsc_accelerator_module_t mca_smsc_accelerator_module;
67+
extern mca_smsc_accelerator_component_t mca_smsc_accelerator_component;
68+
69+
#endif /* OPAL_MCA_SMSC_ACCELERATOR_INTERNAL_H */

0 commit comments

Comments
 (0)