#ifndef _COMMON_H_
#define _COMMON_H_
#include "harness/testHarness.h"
#include "harness/typeWrappers.h"
#include "harness/ThreadPool.h"
#include "host_atomics.h"
#include <vector>
#include <sstream>
#define MAX_DEVICE_THREADS (gHost ? 0U : gMaxDeviceThreads)
#define MAX_HOST_THREADS GetThreadCount()
#define EXECUTE_TEST(error, test)\
error |= test;\
if(error && !gContinueOnError)\
return error;
enum TExplicitAtomicType
{
TYPE_ATOMIC_INT,
TYPE_ATOMIC_UINT,
TYPE_ATOMIC_LONG,
TYPE_ATOMIC_ULONG,
TYPE_ATOMIC_FLOAT,
TYPE_ATOMIC_DOUBLE,
TYPE_ATOMIC_INTPTR_T,
TYPE_ATOMIC_UINTPTR_T,
TYPE_ATOMIC_SIZE_T,
TYPE_ATOMIC_PTRDIFF_T,
TYPE_ATOMIC_FLAG
};
enum TExplicitMemoryScopeType
{
MEMORY_SCOPE_EMPTY,
MEMORY_SCOPE_WORK_GROUP,
MEMORY_SCOPE_DEVICE,
MEMORY_SCOPE_ALL_DEVICES,
MEMORY_SCOPE_ALL_SVM_DEVICES
};
extern bool gHost;
extern bool gOldAPI;
extern bool gContinueOnError;
extern bool gNoGlobalVariables;
extern bool gNoGenericAddressSpace;
extern bool gUseHostPtr;
extern bool gDebug;
extern int gInternalIterations;
extern int gMaxDeviceThreads;
extern cl_device_atomic_capabilities gAtomicMemCap,
gAtomicFenceCap;
extern const char *get_memory_order_type_name(TExplicitMemoryOrderType orderType);
extern const char *get_memory_scope_type_name(TExplicitMemoryScopeType scopeType);
extern cl_int getSupportedMemoryOrdersAndScopes(
cl_device_id device, std::vector<TExplicitMemoryOrderType> &memoryOrders,
std::vector<TExplicitMemoryScopeType> &memoryScopes);
class AtomicTypeInfo
{
public:
TExplicitAtomicType _type;
AtomicTypeInfo(TExplicitAtomicType type): _type(type) {}
cl_uint Size(cl_device_id device);
const char* AtomicTypeName();
const char* RegularTypeName();
const char* AddSubOperandTypeName();
int IsSupported(cl_device_id device);
};
template<typename HostDataType>
class AtomicTypeExtendedInfo : public AtomicTypeInfo
{
public:
AtomicTypeExtendedInfo(TExplicitAtomicType type) : AtomicTypeInfo(type) {}
HostDataType MinValue();
HostDataType MaxValue();
HostDataType SpecialValue(cl_uchar x)
{
HostDataType tmp;
cl_uchar *ptr = (cl_uchar*)&tmp;
for(cl_uint i = 0; i < sizeof(HostDataType)/sizeof(cl_uchar); i++)
ptr[i] = x;
return tmp;
}
HostDataType SpecialValue(cl_ushort x)
{
HostDataType tmp;
cl_ushort *ptr = (cl_ushort*)&tmp;
for(cl_uint i = 0; i < sizeof(HostDataType)/sizeof(cl_ushort); i++)
ptr[i] = x;
return tmp;
}
};
class CTest {
public:
virtual int Execute(cl_device_id deviceID, cl_context context, cl_command_queue queue, int num_elements) = 0;
};
template<typename HostAtomicType, typename HostDataType>
class CBasicTest : CTest
{
public:
typedef struct {
CBasicTest *test;
cl_uint tid;
cl_uint threadCount;
volatile HostAtomicType *destMemory;
HostDataType *oldValues;
} THostThreadContext;
static cl_int HostThreadFunction(cl_uint job_id, cl_uint thread_id, void *userInfo)
{
THostThreadContext *threadContext = ((THostThreadContext*)userInfo)+job_id;
threadContext->test->HostFunction(threadContext->tid, threadContext->threadCount, threadContext->destMemory, threadContext->oldValues);
return 0;
}
CBasicTest(TExplicitAtomicType dataType, bool useSVM) : CTest(),
_maxDeviceThreads(MAX_DEVICE_THREADS),
_dataType(dataType), _useSVM(useSVM), _startValue(255),
_localMemory(false), _declaredInProgram(false),
_usedInFunction(false), _genericAddrSpace(false),
_oldValueCheck(true), _localRefValues(false),
_maxGroupSize(0), _passCount(0), _iterations(gInternalIterations)
{
}
virtual ~CBasicTest()
{
if(_passCount)
log_info(" %u tests executed successfully for %s\n", _passCount, DataType().AtomicTypeName());
}
virtual cl_uint NumResults(cl_uint threadCount, cl_device_id deviceID)
{
return 1;
}
virtual cl_uint NumNonAtomicVariablesPerThread()
{
return 1;
}
virtual bool ExpectedValue(HostDataType &expected, cl_uint threadCount, HostDataType *startRefValues, cl_uint whichDestValue)
{
return false;
}
virtual bool GenerateRefs(cl_uint threadCount, HostDataType *startRefValues, MTdata d)
{
return false;
}
virtual bool VerifyRefs(bool &correct, cl_uint threadCount, HostDataType *refValues, HostAtomicType *finalValues)
{
return false;
}
virtual std::string PragmaHeader(cl_device_id deviceID);
virtual std::string ProgramHeader(cl_uint maxNumDestItems);
virtual std::string FunctionCode();
virtual std::string KernelCode(cl_uint maxNumDestItems);
virtual std::string ProgramCore() = 0;
virtual std::string SingleTestName()
{
std::string testName = LocalMemory() ? "local" : "global";
testName += " ";
testName += DataType().AtomicTypeName();
if(DeclaredInProgram())
{
testName += " declared in program";
}
if(DeclaredInProgram() && UsedInFunction())
testName += ",";
if(UsedInFunction())
{
testName += " used in ";
if(GenericAddrSpace())
testName += "generic ";
testName += "function";
}
return testName;
}
virtual int ExecuteSingleTest(cl_device_id deviceID, cl_context context, cl_command_queue queue);
int ExecuteForEachPointerType(cl_device_id deviceID, cl_context context, cl_command_queue queue)
{
int error = 0;
UsedInFunction(false);
EXECUTE_TEST(error, ExecuteSingleTest(deviceID, context, queue));
UsedInFunction(true);
GenericAddrSpace(false);
EXECUTE_TEST(error, ExecuteSingleTest(deviceID, context, queue));
GenericAddrSpace(true);
EXECUTE_TEST(error, ExecuteSingleTest(deviceID, context, queue));
GenericAddrSpace(false);
return error;
}
int ExecuteForEachDeclarationType(cl_device_id deviceID, cl_context context, cl_command_queue queue)
{
int error = 0;
DeclaredInProgram(false);
EXECUTE_TEST(error, ExecuteForEachPointerType(deviceID, context, queue));
if(!UseSVM())
{
DeclaredInProgram(true);
EXECUTE_TEST(error, ExecuteForEachPointerType(deviceID, context, queue));
}
return error;
}
virtual int ExecuteForEachParameterSet(cl_device_id deviceID, cl_context context, cl_command_queue queue)
{
int error = 0;
if(_maxDeviceThreads > 0 && !UseSVM())
{
LocalMemory(true);
EXECUTE_TEST(error, ExecuteForEachDeclarationType(deviceID, context, queue));
}
if(_maxDeviceThreads+MaxHostThreads() > 0)
{
LocalMemory(false);
EXECUTE_TEST(error, ExecuteForEachDeclarationType(deviceID, context, queue));
}
return error;
}
virtual int Execute(cl_device_id deviceID, cl_context context, cl_command_queue queue, int num_elements)
{
if(sizeof(HostAtomicType) != DataType().Size(deviceID))
{
log_info("Invalid test: Host atomic type size (%u) is different than OpenCL type size (%u)\n", (cl_uint)sizeof(HostAtomicType), DataType().Size(deviceID));
return -1;
}
if(sizeof(HostAtomicType) != sizeof(HostDataType))
{
log_info("Invalid test: Host atomic type size (%u) is different than corresponding type size (%u)\n", (cl_uint)sizeof(HostAtomicType), (cl_uint)sizeof(HostDataType));
return -1;
}
if(UseSVM() && !gUseHostPtr)
{
cl_device_svm_capabilities caps;
cl_int error = clGetDeviceInfo(deviceID, CL_DEVICE_SVM_CAPABILITIES, sizeof(caps), &caps, 0);
test_error(error, "clGetDeviceInfo failed");
if((caps & CL_DEVICE_SVM_ATOMICS) == 0)
{
log_info("\t%s - SVM_ATOMICS not supported\n", DataType().AtomicTypeName());
return 0;
}
}
if(!DataType().IsSupported(deviceID))
{
log_info("\t%s not supported\n", DataType().AtomicTypeName());
if(UseSVM())
return 0;
_maxDeviceThreads = 0;
}
if(_maxDeviceThreads+MaxHostThreads() == 0)
return 0;
return ExecuteForEachParameterSet(deviceID, context, queue);
}
virtual void HostFunction(cl_uint tid, cl_uint threadCount, volatile HostAtomicType *destMemory, HostDataType *oldValues)
{
log_info("Empty thread function %u\n", (cl_uint)tid);
}
AtomicTypeExtendedInfo<HostDataType> DataType() const
{
return AtomicTypeExtendedInfo<HostDataType>(_dataType);
}
cl_uint _maxDeviceThreads;
virtual cl_uint MaxHostThreads()
{
if(UseSVM() || gHost)
return MAX_HOST_THREADS;
else
return 0;
}
int CheckCapabilities(TExplicitMemoryScopeType memoryScope,
TExplicitMemoryOrderType memoryOrder)
{
Differentiation between atomic fence and other atomic operations
does not need to occur here.
The initialisation of this test checks that the minimum required
capabilities are supported by this device.
The following switches allow the test to skip if optional capabilites
are not supported by the device.
*/
switch (memoryScope)
{
case MEMORY_SCOPE_EMPTY: {
break;
}
case MEMORY_SCOPE_WORK_GROUP: {
if ((gAtomicMemCap & CL_DEVICE_ATOMIC_SCOPE_WORK_GROUP) == 0)
{
return TEST_SKIPPED_ITSELF;
}
break;
}
case MEMORY_SCOPE_DEVICE: {
if ((gAtomicMemCap & CL_DEVICE_ATOMIC_SCOPE_DEVICE) == 0)
{
return TEST_SKIPPED_ITSELF;
}
break;
}
case MEMORY_SCOPE_ALL_DEVICES:
case MEMORY_SCOPE_ALL_SVM_DEVICES: {
if ((gAtomicMemCap & CL_DEVICE_ATOMIC_SCOPE_ALL_DEVICES) == 0)
{
return TEST_SKIPPED_ITSELF;
}
break;
}
default: {
log_info("Invalid memory scope\n");
break;
}
}
switch (memoryOrder)
{
case MEMORY_ORDER_EMPTY: {
break;
}
case MEMORY_ORDER_RELAXED: {
if ((gAtomicMemCap & CL_DEVICE_ATOMIC_ORDER_RELAXED) == 0)
{
return TEST_SKIPPED_ITSELF;
}
break;
}
case MEMORY_ORDER_ACQUIRE:
case MEMORY_ORDER_RELEASE:
case MEMORY_ORDER_ACQ_REL: {
if ((gAtomicMemCap & CL_DEVICE_ATOMIC_ORDER_ACQ_REL) == 0)
{
return TEST_SKIPPED_ITSELF;
}
break;
}
case MEMORY_ORDER_SEQ_CST: {
if ((gAtomicMemCap & CL_DEVICE_ATOMIC_ORDER_SEQ_CST) == 0)
{
return TEST_SKIPPED_ITSELF;
}
break;
}
default: {
log_info("Invalid memory order\n");
break;
}
}
return 0;
}
virtual bool SVMDataBufferAllSVMConsistent() {return false;}
bool UseSVM() {return _useSVM;}
void StartValue(HostDataType startValue) {_startValue = startValue;}
HostDataType StartValue() {return _startValue;}
void LocalMemory(bool local) {_localMemory = local;}
bool LocalMemory() {return _localMemory;}
void DeclaredInProgram(bool declaredInProgram) {_declaredInProgram = declaredInProgram;}
bool DeclaredInProgram() {return _declaredInProgram;}
void UsedInFunction(bool local) {_usedInFunction = local;}
bool UsedInFunction() {return _usedInFunction;}
void GenericAddrSpace(bool genericAddrSpace) {_genericAddrSpace = genericAddrSpace;}
bool GenericAddrSpace() {return _genericAddrSpace;}
void OldValueCheck(bool check) {_oldValueCheck = check;}
bool OldValueCheck() {return _oldValueCheck;}
void LocalRefValues(bool localRefValues) {_localRefValues = localRefValues;}
bool LocalRefValues() {return _localRefValues;}
void MaxGroupSize(cl_uint maxGroupSize) {_maxGroupSize = maxGroupSize;}
cl_uint MaxGroupSize() {return _maxGroupSize;}
void CurrentGroupSize(cl_uint currentGroupSize)
{
if(MaxGroupSize() && MaxGroupSize() < currentGroupSize)
_currentGroupSize = MaxGroupSize();
else
_currentGroupSize = currentGroupSize;
}
cl_uint CurrentGroupSize() {return _currentGroupSize;}
virtual cl_uint CurrentGroupNum(cl_uint threadCount)
{
if(threadCount == 0)
return 0;
if(LocalMemory())
return 1;
return threadCount/CurrentGroupSize();
}
cl_int Iterations() {return _iterations;}
std::string IterationsStr() {std::stringstream ss; ss << _iterations; return ss.str();}
private:
const TExplicitAtomicType _dataType;
const bool _useSVM;
HostDataType _startValue;
bool _localMemory;
bool _declaredInProgram;
bool _usedInFunction;
bool _genericAddrSpace;
bool _oldValueCheck;
bool _localRefValues;
cl_uint _maxGroupSize;
cl_uint _currentGroupSize;
cl_uint _passCount;
const cl_int _iterations;
};
template<typename HostAtomicType, typename HostDataType>
class CBasicTestMemOrderScope : public CBasicTest<HostAtomicType, HostDataType>
{
public:
using CBasicTest<HostAtomicType, HostDataType>::LocalMemory;
using CBasicTest<HostAtomicType, HostDataType>::MaxGroupSize;
using CBasicTest<HostAtomicType, HostDataType>::CheckCapabilities;
CBasicTestMemOrderScope(TExplicitAtomicType dataType, bool useSVM = false) : CBasicTest<HostAtomicType, HostDataType>(dataType, useSVM)
{
}
virtual std::string ProgramHeader(cl_uint maxNumDestItems)
{
std::string header;
if(gOldAPI)
{
std::string s = MemoryScope() == MEMORY_SCOPE_EMPTY ? "" : ",s";
header +=
"#define atomic_store_explicit(x,y,o"+s+") atomic_store(x,y)\n"
"#define atomic_load_explicit(x,o"+s+") atomic_load(x)\n"
"#define atomic_exchange_explicit(x,y,o"+s+") atomic_exchange(x,y)\n"
"#define atomic_compare_exchange_strong_explicit(x,y,z,os,of"+s+") atomic_compare_exchange_strong(x,y,z)\n"
"#define atomic_compare_exchange_weak_explicit(x,y,z,os,of"+s+") atomic_compare_exchange_weak(x,y,z)\n"
"#define atomic_fetch_add_explicit(x,y,o"+s+") atomic_fetch_add(x,y)\n"
"#define atomic_fetch_sub_explicit(x,y,o"+s+") atomic_fetch_sub(x,y)\n"
"#define atomic_fetch_or_explicit(x,y,o"+s+") atomic_fetch_or(x,y)\n"
"#define atomic_fetch_xor_explicit(x,y,o"+s+") atomic_fetch_xor(x,y)\n"
"#define atomic_fetch_and_explicit(x,y,o"+s+") atomic_fetch_and(x,y)\n"
"#define atomic_fetch_min_explicit(x,y,o"+s+") atomic_fetch_min(x,y)\n"
"#define atomic_fetch_max_explicit(x,y,o"+s+") atomic_fetch_max(x,y)\n"
"#define atomic_flag_test_and_set_explicit(x,o"+s+") atomic_flag_test_and_set(x)\n"
"#define atomic_flag_clear_explicit(x,o"+s+") atomic_flag_clear(x)\n";
}
return header+CBasicTest<HostAtomicType, HostDataType>::ProgramHeader(maxNumDestItems);
}
virtual std::string SingleTestName()
{
std::string testName = CBasicTest<HostAtomicType, HostDataType>::SingleTestName();
if(MemoryOrder() != MEMORY_ORDER_EMPTY)
{
testName += std::string(", ")+std::string(get_memory_order_type_name(MemoryOrder())).substr(sizeof("memory"));
}
if(MemoryScope() != MEMORY_SCOPE_EMPTY)
{
testName += std::string(", ")+std::string(get_memory_scope_type_name(MemoryScope())).substr(sizeof("memory"));
}
return testName;
}
virtual int ExecuteSingleTest(cl_device_id deviceID, cl_context context, cl_command_queue queue)
{
if(LocalMemory() &&
MemoryScope() != MEMORY_SCOPE_EMPTY &&
MemoryScope() != MEMORY_SCOPE_WORK_GROUP)
return 0;
if(MemoryScope() == MEMORY_SCOPE_DEVICE)
MaxGroupSize(16);
else
MaxGroupSize(0);
if (CheckCapabilities(MemoryScope(), MemoryOrder()) == TEST_SKIPPED_ITSELF)
return 0;
return CBasicTest<HostAtomicType, HostDataType>::ExecuteSingleTest(deviceID, context, queue);
}
virtual int ExecuteForEachParameterSet(cl_device_id deviceID, cl_context context, cl_command_queue queue)
{
std::vector<TExplicitMemoryOrderType> memoryOrder;
std::vector<TExplicitMemoryScopeType> memoryScope;
int error = 0;
test_error_ret(
getSupportedMemoryOrdersAndScopes(deviceID, memoryOrder, memoryScope),
"getSupportedMemoryOrdersAndScopes failed\n", TEST_FAIL);
for(unsigned oi = 0; oi < memoryOrder.size(); oi++)
{
for(unsigned si = 0; si < memoryScope.size(); si++)
{
if(memoryOrder[oi] == MEMORY_ORDER_EMPTY && memoryScope[si] != MEMORY_SCOPE_EMPTY)
continue;
MemoryOrder(memoryOrder[oi]);
MemoryScope(memoryScope[si]);
EXECUTE_TEST(error, (CBasicTest<HostAtomicType, HostDataType>::ExecuteForEachParameterSet(deviceID, context, queue)));
}
}
return error;
}
void MemoryOrder(TExplicitMemoryOrderType memoryOrder) {_memoryOrder = memoryOrder;}
TExplicitMemoryOrderType MemoryOrder() {return _memoryOrder;}
std::string MemoryOrderStr()
{
if(MemoryOrder() != MEMORY_ORDER_EMPTY)
return std::string(", ")+get_memory_order_type_name(MemoryOrder());
return "";
}
void MemoryScope(TExplicitMemoryScopeType memoryScope) {_memoryScope = memoryScope;}
TExplicitMemoryScopeType MemoryScope() {return _memoryScope;}
std::string MemoryScopeStr()
{
if(MemoryScope() != MEMORY_SCOPE_EMPTY)
return std::string(", ")+get_memory_scope_type_name(MemoryScope());
return "";
}
std::string MemoryOrderScopeStr()
{
return MemoryOrderStr()+MemoryScopeStr();
}
virtual cl_uint CurrentGroupNum(cl_uint threadCount)
{
if(MemoryScope() == MEMORY_SCOPE_WORK_GROUP)
return 1;
return CBasicTest<HostAtomicType, HostDataType>::CurrentGroupNum(threadCount);
}
virtual cl_uint MaxHostThreads()
{
if (MemoryScope() == MEMORY_SCOPE_ALL_DEVICES
|| MemoryScope() == MEMORY_SCOPE_ALL_SVM_DEVICES || gHost)
{
return CBasicTest<HostAtomicType, HostDataType>::MaxHostThreads();
}
else
{
return 0;
}
}
private:
TExplicitMemoryOrderType _memoryOrder;
TExplicitMemoryScopeType _memoryScope;
};
template<typename HostAtomicType, typename HostDataType>
class CBasicTestMemOrder2Scope : public CBasicTestMemOrderScope<HostAtomicType, HostDataType>
{
public:
using CBasicTestMemOrderScope<HostAtomicType, HostDataType>::LocalMemory;
using CBasicTestMemOrderScope<HostAtomicType, HostDataType>::MemoryOrder;
using CBasicTestMemOrderScope<HostAtomicType, HostDataType>::MemoryScope;
using CBasicTestMemOrderScope<HostAtomicType, HostDataType>::MemoryOrderStr;
using CBasicTestMemOrderScope<HostAtomicType, HostDataType>::MemoryScopeStr;
using CBasicTest<HostAtomicType, HostDataType>::CheckCapabilities;
CBasicTestMemOrder2Scope(TExplicitAtomicType dataType, bool useSVM = false) : CBasicTestMemOrderScope<HostAtomicType, HostDataType>(dataType, useSVM)
{
}
virtual std::string SingleTestName()
{
std::string testName = CBasicTest<HostAtomicType, HostDataType>::SingleTestName();
if(MemoryOrder() != MEMORY_ORDER_EMPTY)
testName += std::string(", ")+std::string(get_memory_order_type_name(MemoryOrder())).substr(sizeof("memory"));
if(MemoryOrder2() != MEMORY_ORDER_EMPTY)
testName += std::string(", ")+std::string(get_memory_order_type_name(MemoryOrder2())).substr(sizeof("memory"));
if(MemoryScope() != MEMORY_SCOPE_EMPTY)
testName += std::string(", ")+std::string(get_memory_scope_type_name(MemoryScope())).substr(sizeof("memory"));
return testName;
}
virtual int ExecuteForEachParameterSet(cl_device_id deviceID, cl_context context, cl_command_queue queue)
{
std::vector<TExplicitMemoryOrderType> memoryOrder;
std::vector<TExplicitMemoryScopeType> memoryScope;
int error = 0;
test_error_ret(
getSupportedMemoryOrdersAndScopes(deviceID, memoryOrder, memoryScope),
"getSupportedMemoryOrdersAndScopes failed\n", TEST_FAIL);
for(unsigned oi = 0; oi < memoryOrder.size(); oi++)
{
for(unsigned o2i = 0; o2i < memoryOrder.size(); o2i++)
{
for(unsigned si = 0; si < memoryScope.size(); si++)
{
if((memoryOrder[oi] == MEMORY_ORDER_EMPTY || memoryOrder[o2i] == MEMORY_ORDER_EMPTY)
&& memoryOrder[oi] != memoryOrder[o2i])
continue;
if((memoryOrder[oi] == MEMORY_ORDER_EMPTY || memoryOrder[o2i] == MEMORY_ORDER_EMPTY)
&& memoryScope[si] != MEMORY_SCOPE_EMPTY)
continue;
MemoryOrder(memoryOrder[oi]);
MemoryOrder2(memoryOrder[o2i]);
MemoryScope(memoryScope[si]);
if (CheckCapabilities(MemoryScope(), MemoryOrder())
== TEST_SKIPPED_ITSELF)
continue;
if (CheckCapabilities(MemoryScope(), MemoryOrder2())
== TEST_SKIPPED_ITSELF)
continue;
EXECUTE_TEST(error, (CBasicTest<HostAtomicType, HostDataType>::ExecuteForEachParameterSet(deviceID, context, queue)));
}
}
}
return error;
}
void MemoryOrder2(TExplicitMemoryOrderType memoryOrderFail) {_memoryOrder2 = memoryOrderFail;}
TExplicitMemoryOrderType MemoryOrder2() {return _memoryOrder2;}
std::string MemoryOrderFailStr()
{
if(MemoryOrder2() != MEMORY_ORDER_EMPTY)
return std::string(", ")+get_memory_order_type_name(MemoryOrder2());
return "";
}
std::string MemoryOrderScope()
{
return MemoryOrderStr()+MemoryOrderFailStr()+MemoryScopeStr();
}
private:
TExplicitMemoryOrderType _memoryOrder2;
};
template<typename HostAtomicType, typename HostDataType>
std::string CBasicTest<HostAtomicType, HostDataType>::PragmaHeader(cl_device_id deviceID)
{
std::string pragma;
if(gOldAPI)
{
pragma += "#pragma OPENCL EXTENSION cl_khr_local_int32_base_atomics : enable\n";
pragma += "#pragma OPENCL EXTENSION cl_khr_local_int32_extended_atomics : enable\n";
pragma += "#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics : enable\n";
pragma += "#pragma OPENCL EXTENSION cl_khr_global_int32_extended_atomics : enable\n";
}
if(DataType().Size(deviceID) == 8)
{
pragma += "#pragma OPENCL EXTENSION cl_khr_int64_base_atomics : enable\n";
pragma += "#pragma OPENCL EXTENSION cl_khr_int64_extended_atomics : enable\n";
}
if(_dataType == TYPE_ATOMIC_DOUBLE)
pragma += "#pragma OPENCL EXTENSION cl_khr_fp64 : enable\n";
return pragma;
}
template<typename HostAtomicType, typename HostDataType>
std::string CBasicTest<HostAtomicType, HostDataType>::ProgramHeader(cl_uint maxNumDestItems)
{
std::string header;
std::string aTypeName = DataType().AtomicTypeName();
std::string cTypeName = DataType().RegularTypeName();
std::string argListForKernel;
std::string argListForFunction;
std::string argListNoTypes;
std::string functionPrototype;
std::string addressSpace = LocalMemory() ? "__local " : "__global ";
if(gOldAPI)
{
header += std::string("#define ")+aTypeName+" "+cTypeName+"\n"
"#define atomic_store(x,y) (*(x) = y)\n"
"#define atomic_load(x) (*(x))\n"
"#define ATOMIC_VAR_INIT(x) (x)\n"
"#define ATOMIC_FLAG_INIT 0\n"
"#define atomic_init(x,y) atomic_store(x,y)\n";
if(aTypeName == "atomic_float")
header += "#define atomic_exchange(x,y) atomic_xchg(x,y)\n";
else if(aTypeName == "atomic_double")
header += "double atomic_exchange(volatile "+addressSpace+"atomic_double *x, double y)\n"
"{\n"
" long tmp = *(long*)&y, res;\n"
" volatile "+addressSpace+"long *tmpA = (volatile "+addressSpace+"long)x;\n"
" res = atom_xchg(tmpA,tmp);\n"
" return *(double*)&res;\n"
"}\n";
else
header += "#define atomic_exchange(x,y) atom_xchg(x,y)\n";
if(aTypeName != "atomic_float" && aTypeName != "atomic_double")
header +=
"bool atomic_compare_exchange_strong(volatile "+addressSpace+" "+aTypeName+" *a, "+cTypeName+" *expected, "+cTypeName+" desired)\n"
"{\n"
" "+cTypeName+" old = atom_cmpxchg(a, *expected, desired);\n"
" if(old == *expected)\n"
" return true;\n"
" *expected = old;\n"
" return false;\n"
"}\n"
"#define atomic_compare_exchange_weak atomic_compare_exchange_strong\n";
header +=
"#define atomic_fetch_add(x,y) atom_add(x,y)\n"
"#define atomic_fetch_sub(x,y) atom_sub(x,y)\n"
"#define atomic_fetch_or(x,y) atom_or(x,y)\n"
"#define atomic_fetch_xor(x,y) atom_xor(x,y)\n"
"#define atomic_fetch_and(x,y) atom_and(x,y)\n"
"#define atomic_fetch_min(x,y) atom_min(x,y)\n"
"#define atomic_fetch_max(x,y) atom_max(x,y)\n"
"#define atomic_flag_test_and_set(x) atomic_exchange(x,1)\n"
"#define atomic_flag_clear(x) atomic_store(x,0)\n"
"\n";
}
if(!LocalMemory() && DeclaredInProgram())
{
header += "__global volatile atomic_uint finishedThreads = ATOMIC_VAR_INIT(0);\n";
std::stringstream ss;
ss << maxNumDestItems;
header += std::string("__global volatile ")+aTypeName+" destMemory["+ss.str()+"] = {\n";
ss.str("");
ss << _startValue;
for(cl_uint i = 0; i < maxNumDestItems; i++)
{
if(aTypeName == "atomic_flag")
header += " ATOMIC_FLAG_INIT";
else
header += " ATOMIC_VAR_INIT("+ss.str()+")";
if(i+1 < maxNumDestItems)
header += ",";
header += "\n";
}
header+=
"};\n"
"\n";
}
return header;
}
template<typename HostAtomicType, typename HostDataType>
std::string CBasicTest<HostAtomicType, HostDataType>::FunctionCode()
{
if(!UsedInFunction())
return "";
std::string addressSpace = LocalMemory() ? "__local " : "__global ";
std::string code = "void test_atomic_function(uint tid, uint threadCount, uint numDestItems, volatile ";
if(!GenericAddrSpace())
code += addressSpace;
code += std::string(DataType().AtomicTypeName())+" *destMemory, __global "+DataType().RegularTypeName()+
" *oldValues";
if(LocalRefValues())
code += std::string(", __local ")+DataType().RegularTypeName()+" *localValues";
code += ")\n"
"{\n";
code += ProgramCore();
code += "}\n"
"\n";
return code;
}
template<typename HostAtomicType, typename HostDataType>
std::string CBasicTest<HostAtomicType, HostDataType>::KernelCode(cl_uint maxNumDestItems)
{
std::string aTypeName = DataType().AtomicTypeName();
std::string cTypeName = DataType().RegularTypeName();
std::string addressSpace = LocalMemory() ? "__local " : "__global ";
std::string code = "__kernel void test_atomic_kernel(uint threadCount, uint numDestItems, ";
if(LocalMemory())
{
code += std::string("__global ")+cTypeName+" *finalDest, __global "+cTypeName+" *oldValues,"
" volatile "+addressSpace+aTypeName+" *"+(DeclaredInProgram() ? "notUsed" : "")+"destMemory";
}
else
{
code += "volatile "+addressSpace+(DeclaredInProgram() ? (cTypeName+" *finalDest") : (aTypeName+" *destMemory"))+
", __global "+cTypeName+" *oldValues";
}
if(LocalRefValues())
code += std::string(", __local ")+cTypeName+" *localValues";
code += ")\n"
"{\n";
if(LocalMemory() && DeclaredInProgram())
{
std::stringstream ss;
ss << maxNumDestItems;
code += std::string(" __local volatile ")+aTypeName+" destMemory["+ss.str()+"];\n";
}
code += " uint tid = get_global_id(0);\n"
"\n";
if(LocalMemory())
{
code += R"(
// initialize atomics not reachable from host (first thread
// is doing this, other threads are waiting on barrier)
if(get_local_id(0) == 0)
for(uint dstItemIdx = 0; dstItemIdx < numDestItems; dstItemIdx++)
{)";
if (aTypeName == "atomic_flag")
{
code += R"(
if(finalDest[dstItemIdx])
atomic_flag_test_and_set_explicit(destMemory+dstItemIdx,
memory_order_relaxed,
memory_scope_work_group);
else
atomic_flag_clear_explicit(destMemory+dstItemIdx,
memory_order_relaxed,
memory_scope_work_group);)";
}
else
{
code += R"(
atomic_store_explicit(destMemory+dstItemIdx,
finalDest[dstItemIdx],
memory_order_relaxed,
memory_scope_work_group);)";
}
code +=
" }\n"
" barrier(CLK_LOCAL_MEM_FENCE);\n"
"\n";
}
if (LocalRefValues())
{
code +=
" // Copy input reference values into local memory\n";
if (NumNonAtomicVariablesPerThread() == 1)
code += " localValues[get_local_id(0)] = oldValues[tid];\n";
else
{
std::stringstream ss;
ss << NumNonAtomicVariablesPerThread();
code +=
" for(uint rfId = 0; rfId < " + ss.str() + "; rfId++)\n"
" localValues[get_local_id(0)*" + ss.str() + "+rfId] = oldValues[tid*" + ss.str() + "+rfId];\n";
}
code +=
" barrier(CLK_LOCAL_MEM_FENCE);\n"
"\n";
}
if (UsedInFunction())
code += std::string(" test_atomic_function(tid, threadCount, numDestItems, destMemory, oldValues")+
(LocalRefValues() ? ", localValues" : "")+");\n";
else
code += ProgramCore();
code += "\n";
if (LocalRefValues())
{
code +=
" // Copy local reference values into output array\n"
" barrier(CLK_LOCAL_MEM_FENCE);\n";
if (NumNonAtomicVariablesPerThread() == 1)
code += " oldValues[tid] = localValues[get_local_id(0)];\n";
else
{
std::stringstream ss;
ss << NumNonAtomicVariablesPerThread();
code +=
" for(uint rfId = 0; rfId < " + ss.str() + "; rfId++)\n"
" oldValues[tid*" + ss.str() + "+rfId] = localValues[get_local_id(0)*" + ss.str() + "+rfId];\n";
}
code += "\n";
}
if(LocalMemory() || DeclaredInProgram())
{
code += " // Copy final values to host reachable buffer\n";
if(LocalMemory())
code +=
" barrier(CLK_LOCAL_MEM_FENCE);\n"
" if(get_local_id(0) == 0) // first thread in workgroup\n";
else
code += R"(
if(atomic_fetch_add_explicit(&finishedThreads, 1u,
memory_order_relaxed,
memory_scope_work_group)
== get_global_size(0)-1) // last finished thread
)";
code +=
" for(uint dstItemIdx = 0; dstItemIdx < numDestItems; dstItemIdx++)\n";
if(aTypeName == "atomic_flag")
{
code += R"(
finalDest[dstItemIdx] =
atomic_flag_test_and_set_explicit(destMemory+dstItemIdx,
memory_order_relaxed,
memory_scope_work_group);)";
}
else
{
code += R"(
finalDest[dstItemIdx] =
atomic_load_explicit(destMemory+dstItemIdx,
memory_order_relaxed,
memory_scope_work_group);)";
}
}
code += "}\n"
"\n";
return code;
}
template <typename HostAtomicType, typename HostDataType>
int CBasicTest<HostAtomicType, HostDataType>::ExecuteSingleTest(cl_device_id deviceID, cl_context context, cl_command_queue queue)
{
int error;
clProgramWrapper program;
clKernelWrapper kernel;
size_t threadNum[1];
clMemWrapper streams[2];
std::vector<HostAtomicType> destItems;
HostAtomicType *svmAtomicBuffer = 0;
std::vector<HostDataType> refValues, startRefValues;
HostDataType *svmDataBuffer = 0;
cl_uint deviceThreadCount, hostThreadCount, threadCount;
size_t groupSize = 0;
std::string programSource;
const char *programLine;
MTdata d;
size_t typeSize = DataType().Size(deviceID);
deviceThreadCount = _maxDeviceThreads;
hostThreadCount = MaxHostThreads();
threadCount = deviceThreadCount+hostThreadCount;
log_info("\t%s...\n", SingleTestName().c_str());
if(!LocalMemory() && DeclaredInProgram() && gNoGlobalVariables)
{
log_info("\t\tTest disabled\n");
return 0;
}
if(UsedInFunction() && GenericAddrSpace() && gNoGenericAddressSpace)
{
log_info("\t\tTest disabled\n");
return 0;
}
error = clGetDeviceInfo(deviceID, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(groupSize), &groupSize, NULL);
test_error(error, "Unable to obtain max work group size for device");
CurrentGroupSize((cl_uint)groupSize);
if(CurrentGroupSize() > deviceThreadCount)
CurrentGroupSize(deviceThreadCount);
if(CurrentGroupNum(deviceThreadCount) == 1 || gOldAPI)
deviceThreadCount = CurrentGroupSize()*CurrentGroupNum(deviceThreadCount);
threadCount = deviceThreadCount+hostThreadCount;
cl_uint numDestItems = NumResults(threadCount, deviceID);
if(deviceThreadCount > 0)
{
while ((CurrentGroupSize() > 1))
{
if (kernel) clReleaseKernel(kernel);
if (program) clReleaseProgram(program);
programSource = PragmaHeader(deviceID) + ProgramHeader(numDestItems)
+ FunctionCode() + KernelCode(numDestItems);
programLine = programSource.c_str();
if (create_single_kernel_helper_with_build_options(
context, &program, &kernel, 1, &programLine,
"test_atomic_kernel", gOldAPI ? "" : nullptr))
{
return -1;
}
error = clGetKernelWorkGroupInfo(kernel, deviceID,
CL_KERNEL_WORK_GROUP_SIZE,
sizeof(groupSize), &groupSize, NULL);
test_error(error,
"Unable to obtain max work group size for device and "
"kernel combo");
if (LocalMemory())
{
cl_ulong usedLocalMemory;
cl_ulong totalLocalMemory;
cl_uint maxWorkGroupSize;
error = clGetKernelWorkGroupInfo(
kernel, deviceID, CL_KERNEL_LOCAL_MEM_SIZE,
sizeof(usedLocalMemory), &usedLocalMemory, NULL);
test_error(error, "clGetKernelWorkGroupInfo failed");
error = clGetDeviceInfo(deviceID, CL_DEVICE_LOCAL_MEM_SIZE,
sizeof(totalLocalMemory),
&totalLocalMemory, NULL);
test_error(error, "clGetDeviceInfo failed");
maxWorkGroupSize =
((totalLocalMemory - usedLocalMemory) / typeSize);
if (maxWorkGroupSize < groupSize) groupSize = maxWorkGroupSize;
}
if (CurrentGroupSize() <= groupSize)
break;
else
CurrentGroupSize(CurrentGroupSize() / 2);
}
if(CurrentGroupSize() > deviceThreadCount)
CurrentGroupSize(deviceThreadCount);
if(CurrentGroupNum(deviceThreadCount) == 1 || gOldAPI)
deviceThreadCount = CurrentGroupSize()*CurrentGroupNum(deviceThreadCount);
threadCount = deviceThreadCount+hostThreadCount;
}
if (gDebug)
{
log_info("Program source:\n");
log_info("%s\n", programLine);
}
if(deviceThreadCount > 0)
log_info("\t\t(thread count %u, group size %u)\n", deviceThreadCount, CurrentGroupSize());
if(hostThreadCount > 0)
log_info("\t\t(host threads %u)\n", hostThreadCount);
refValues.resize(threadCount*NumNonAtomicVariablesPerThread());
d = init_genrand(gRandomSeed);
startRefValues.resize(threadCount*NumNonAtomicVariablesPerThread());
if(GenerateRefs(threadCount, &startRefValues[0], d))
{
memcpy(&refValues[0], &startRefValues[0], sizeof(HostDataType)*threadCount*NumNonAtomicVariablesPerThread());
}
else
{
startRefValues.resize(0);
}
free_mtdata(d);
d = NULL;
numDestItems = NumResults(threadCount, deviceID);
destItems.resize(numDestItems);
for(cl_uint i = 0; i < numDestItems; i++)
destItems[i] = _startValue;
if(UseSVM())
{
if(gUseHostPtr)
svmAtomicBuffer = (HostAtomicType*)malloc(typeSize * numDestItems);
else
svmAtomicBuffer = (HostAtomicType*)clSVMAlloc(context, CL_MEM_SVM_FINE_GRAIN_BUFFER | CL_MEM_SVM_ATOMICS, typeSize * numDestItems, 0);
if(!svmAtomicBuffer)
{
log_error("ERROR: clSVMAlloc failed!\n");
return -1;
}
memcpy(svmAtomicBuffer, &destItems[0], typeSize * numDestItems);
streams[0] = clCreateBuffer(context, CL_MEM_USE_HOST_PTR,
typeSize * numDestItems, svmAtomicBuffer, NULL);
}
else
{
streams[0] = clCreateBuffer(context, CL_MEM_COPY_HOST_PTR,
typeSize * numDestItems, &destItems[0], NULL);
}
if (!streams[0])
{
log_error("ERROR: Creating output array failed!\n");
return -1;
}
if(UseSVM())
{
if(gUseHostPtr)
svmDataBuffer = (HostDataType*)malloc(typeSize*threadCount*NumNonAtomicVariablesPerThread());
else
svmDataBuffer = (HostDataType*)clSVMAlloc(context, CL_MEM_SVM_FINE_GRAIN_BUFFER | (SVMDataBufferAllSVMConsistent() ? CL_MEM_SVM_ATOMICS : 0), typeSize*threadCount*NumNonAtomicVariablesPerThread(), 0);
if(!svmDataBuffer)
{
log_error("ERROR: clSVMAlloc failed!\n");
return -1;
}
if(startRefValues.size())
memcpy(svmDataBuffer, &startRefValues[0], typeSize*threadCount*NumNonAtomicVariablesPerThread());
streams[1] = clCreateBuffer(context, CL_MEM_USE_HOST_PTR,
typeSize * threadCount
* NumNonAtomicVariablesPerThread(),
svmDataBuffer, NULL);
}
else
{
streams[1] = clCreateBuffer(
context,
((startRefValues.size() ? CL_MEM_COPY_HOST_PTR : CL_MEM_READ_WRITE)),
typeSize * threadCount * NumNonAtomicVariablesPerThread(),
startRefValues.size() ? &startRefValues[0] : 0, NULL);
}
if (!streams[1])
{
log_error("ERROR: Creating reference array failed!\n");
return -1;
}
if(deviceThreadCount > 0)
{
cl_uint argInd = 0;
error = clSetKernelArg(kernel, argInd++, sizeof(threadCount), &threadCount);
test_error(error, "Unable to set kernel argument");
error = clSetKernelArg(kernel, argInd++, sizeof(numDestItems), &numDestItems);
test_error(error, "Unable to set indexed kernel argument");
error = clSetKernelArg(kernel, argInd++, sizeof(streams[0]), &streams[0]);
test_error(error, "Unable to set indexed kernel arguments");
error = clSetKernelArg(kernel, argInd++, sizeof(streams[1]), &streams[1]);
test_error(error, "Unable to set indexed kernel arguments");
if(LocalMemory())
{
error = clSetKernelArg(kernel, argInd++, typeSize * numDestItems, NULL);
test_error(error, "Unable to set indexed local kernel argument");
}
if(LocalRefValues())
{
error = clSetKernelArg(kernel, argInd++, LocalRefValues() ? typeSize*CurrentGroupSize()*NumNonAtomicVariablesPerThread() : 1, NULL);
test_error(error, "Unable to set indexed kernel argument");
}
}
std::vector<THostThreadContext> hostThreadContexts(hostThreadCount);
for(unsigned int t = 0; t < hostThreadCount; t++)
{
hostThreadContexts[t].test = this;
hostThreadContexts[t].tid = deviceThreadCount+t;
hostThreadContexts[t].threadCount = threadCount;
hostThreadContexts[t].destMemory = UseSVM() ? svmAtomicBuffer : &destItems[0];
hostThreadContexts[t].oldValues = UseSVM() ? svmDataBuffer : &refValues[0];
}
if(deviceThreadCount > 0)
{
threadNum[0] = deviceThreadCount;
groupSize = CurrentGroupSize();
error = clEnqueueNDRangeKernel(queue, kernel, 1, NULL, threadNum, &groupSize, 0, NULL, NULL);
test_error(error, "Unable to execute test kernel");
error = clFlush(queue);
test_error(error, "clFlush failed");
}
if(hostThreadCount > 0)
ThreadPool_Do(HostThreadFunction, hostThreadCount, &hostThreadContexts[0]);
if(UseSVM())
{
error = clFinish(queue);
test_error(error, "clFinish failed");
memcpy(&destItems[0], svmAtomicBuffer, typeSize*numDestItems);
memcpy(&refValues[0], svmDataBuffer, typeSize*threadCount*NumNonAtomicVariablesPerThread());
}
else
{
if(deviceThreadCount > 0)
{
error = clEnqueueReadBuffer(queue, streams[0], CL_TRUE, 0, typeSize * numDestItems, &destItems[0], 0, NULL, NULL);
test_error(error, "Unable to read result value!");
error = clEnqueueReadBuffer(queue, streams[1], CL_TRUE, 0, typeSize * deviceThreadCount*NumNonAtomicVariablesPerThread(), &refValues[0], 0, NULL, NULL);
test_error(error, "Unable to read reference values!");
}
}
bool dataVerified = false;
for(cl_uint i = 0; i < numDestItems; i++)
{
HostDataType expected;
if(!ExpectedValue(expected, threadCount, startRefValues.size() ? &startRefValues[0] : 0, i))
break;
if(expected != destItems[i])
{
std::stringstream logLine;
logLine << "ERROR: Result " << i << " from kernel does not validate! (should be " << expected << ", was " << destItems[i] << ")\n";
log_error("%s", logLine.str().c_str());
for(i = 0; i < threadCount; i++)
{
logLine.str("");
logLine << " --- " << i << " - ";
if(startRefValues.size())
logLine << startRefValues[i] << " -> " << refValues[i];
else
logLine << refValues[i];
logLine << " --- ";
if(i < numDestItems)
logLine << destItems[i];
logLine << "\n";
log_info("%s", logLine.str().c_str());
}
if(!gDebug)
{
log_info("Program source:\n");
log_info("%s\n", programLine);
}
return -1;
}
dataVerified = true;
}
bool dataCorrect = false;
if(VerifyRefs(dataCorrect, threadCount, &refValues[0], &destItems[0]))
{
if(!dataCorrect)
{
log_error("ERROR: Reference values did not validate!\n");
std::stringstream logLine;
for(cl_uint i = 0; i < threadCount; i++)
for (cl_uint j = 0; j < NumNonAtomicVariablesPerThread(); j++)
{
logLine.str("");
logLine << " --- " << i << " - " << refValues[i*NumNonAtomicVariablesPerThread()+j] << " --- ";
if(j == 0 && i < numDestItems)
logLine << destItems[i];
logLine << "\n";
log_info("%s", logLine.str().c_str());
}
if(!gDebug)
{
log_info("Program source:\n");
log_info("%s\n", programLine);
}
return -1;
}
}
else if(!dataVerified)
{
log_error("ERROR: Test doesn't check total or refs; no values are verified!\n");
return -1;
}
if(OldValueCheck() &&
!(DeclaredInProgram() && !LocalMemory()))
{
for(size_t i = 0; i < numDestItems; i++)
destItems[i] = _startValue;
refValues[0] = 0;
if(deviceThreadCount > 0)
{
error = clEnqueueWriteBuffer(queue, streams[0], CL_TRUE, 0, typeSize * numDestItems, &destItems[0], 0, NULL, NULL);
test_error(error, "Unable to write starting values!");
threadNum[0] = 1;
error = clEnqueueNDRangeKernel(queue, kernel, 1, NULL, threadNum, threadNum, 0, NULL, NULL);
test_error(error, "Unable to execute test kernel");
error = clEnqueueReadBuffer(queue, streams[1], CL_TRUE, 0, typeSize, &refValues[0], 0, NULL, NULL);
test_error(error, "Unable to read reference values!");
}
else
{
HostFunction(0, 1, &destItems[0], &refValues[0]);
}
if(refValues[0] != _startValue)
{
std::stringstream logLine;
logLine << "ERROR: atomic function operated correctly but did NOT return correct 'old' value "
" (should have been " << destItems[0] << ", returned " << refValues[0] << ")!\n";
log_error("%s", logLine.str().c_str());
if(!gDebug)
{
log_info("Program source:\n");
log_info("%s\n", programLine);
}
return -1;
}
}
if(UseSVM())
{
error = clReleaseMemObject(streams[0]);
streams[0] = 0;
test_error(error, "clReleaseMemObject failed");
if(gUseHostPtr)
free(svmAtomicBuffer);
else
clSVMFree(context, svmAtomicBuffer);
error = clReleaseMemObject(streams[1]);
streams[1] = 0;
test_error(error, "clReleaseMemObject failed");
if(gUseHostPtr)
free(svmDataBuffer);
else
clSVMFree(context, svmDataBuffer);
}
_passCount++;
return 0;
}
#endif