0478-drm-vc4-Add-support-for-branching-in-shader-validati.patch 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475
  1. From f2ba2314600620134530571d3b8b22de2ad5745b Mon Sep 17 00:00:00 2001
  2. From: Eric Anholt <eric@anholt.net>
  3. Date: Sat, 2 Jul 2016 12:17:10 -0700
  4. Subject: [PATCH] drm/vc4: Add support for branching in shader validation.
  5. We're already checking that branch instructions are between the start
  6. of the shader and the proper PROG_END sequence. The other thing we
  7. need to make branching safe is to verify that the shader doesn't read
  8. past the end of the uniforms stream.
  9. To do that, we require that at any basic block reading uniforms have
  10. the following instructions:
  11. load_imm temp, <next offset within uniform stream>
  12. add unif_addr, temp, unif
  13. The instructions are generated by userspace, and the kernel verifies
  14. that the load_imm is of the expected offset, and that the add adds it
  15. to a uniform. We track which uniform in the stream that is, and at
  16. draw call time fix up the uniform stream to have the address of the
  17. start of the shader's uniforms at that location.
  18. Signed-off-by: Eric Anholt <eric@anholt.net>
  19. (cherry picked from commit 6d45c81d229d71da54d374143e7d6abad4c0cf31)
  20. ---
  21. drivers/gpu/drm/vc4/vc4_drv.h | 3 +
  22. drivers/gpu/drm/vc4/vc4_qpu_defines.h | 3 +
  23. drivers/gpu/drm/vc4/vc4_validate.c | 13 +-
  24. drivers/gpu/drm/vc4/vc4_validate_shaders.c | 281 +++++++++++++++++++++++++++--
  25. 4 files changed, 283 insertions(+), 17 deletions(-)
  26. --- a/drivers/gpu/drm/vc4/vc4_drv.h
  27. +++ b/drivers/gpu/drm/vc4/vc4_drv.h
  28. @@ -363,6 +363,9 @@ struct vc4_validated_shader_info {
  29. uint32_t uniforms_src_size;
  30. uint32_t num_texture_samples;
  31. struct vc4_texture_sample_info *texture_samples;
  32. +
  33. + uint32_t num_uniform_addr_offsets;
  34. + uint32_t *uniform_addr_offsets;
  35. };
  36. /**
  37. --- a/drivers/gpu/drm/vc4/vc4_qpu_defines.h
  38. +++ b/drivers/gpu/drm/vc4/vc4_qpu_defines.h
  39. @@ -270,6 +270,9 @@ enum qpu_unpack_r4 {
  40. #define QPU_OP_ADD_SHIFT 24
  41. #define QPU_OP_ADD_MASK QPU_MASK(28, 24)
  42. +#define QPU_LOAD_IMM_SHIFT 0
  43. +#define QPU_LOAD_IMM_MASK QPU_MASK(31, 0)
  44. +
  45. #define QPU_BRANCH_TARGET_SHIFT 0
  46. #define QPU_BRANCH_TARGET_MASK QPU_MASK(31, 0)
  47. --- a/drivers/gpu/drm/vc4/vc4_validate.c
  48. +++ b/drivers/gpu/drm/vc4/vc4_validate.c
  49. @@ -802,7 +802,7 @@ validate_gl_shader_rec(struct drm_device
  50. uint32_t src_offset = *(uint32_t *)(pkt_u + o);
  51. uint32_t *texture_handles_u;
  52. void *uniform_data_u;
  53. - uint32_t tex;
  54. + uint32_t tex, uni;
  55. *(uint32_t *)(pkt_v + o) = bo[i]->paddr + src_offset;
  56. @@ -840,6 +840,17 @@ validate_gl_shader_rec(struct drm_device
  57. }
  58. }
  59. + /* Fill in the uniform slots that need this shader's
  60. + * start-of-uniforms address (used for resetting the uniform
  61. + * stream in the presence of control flow).
  62. + */
  63. + for (uni = 0;
  64. + uni < validated_shader->num_uniform_addr_offsets;
  65. + uni++) {
  66. + uint32_t o = validated_shader->uniform_addr_offsets[uni];
  67. + ((uint32_t *)exec->uniforms_v)[o] = exec->uniforms_p;
  68. + }
  69. +
  70. *(uint32_t *)(pkt_v + o + 4) = exec->uniforms_p;
  71. exec->uniforms_u += validated_shader->uniforms_src_size;
  72. --- a/drivers/gpu/drm/vc4/vc4_validate_shaders.c
  73. +++ b/drivers/gpu/drm/vc4/vc4_validate_shaders.c
  74. @@ -39,6 +39,8 @@
  75. #include "vc4_drv.h"
  76. #include "vc4_qpu_defines.h"
  77. +#define LIVE_REG_COUNT (32 + 32 + 4)
  78. +
  79. struct vc4_shader_validation_state {
  80. /* Current IP being validated. */
  81. uint32_t ip;
  82. @@ -57,8 +59,9 @@ struct vc4_shader_validation_state {
  83. *
  84. * This is used for the validation of direct address memory reads.
  85. */
  86. - uint32_t live_min_clamp_offsets[32 + 32 + 4];
  87. - bool live_max_clamp_regs[32 + 32 + 4];
  88. + uint32_t live_min_clamp_offsets[LIVE_REG_COUNT];
  89. + bool live_max_clamp_regs[LIVE_REG_COUNT];
  90. + uint32_t live_immediates[LIVE_REG_COUNT];
  91. /* Bitfield of which IPs are used as branch targets.
  92. *
  93. @@ -66,6 +69,20 @@ struct vc4_shader_validation_state {
  94. * points and clearing the texturing/clamping state.
  95. */
  96. unsigned long *branch_targets;
  97. +
  98. + /* Set when entering a basic block, and cleared when the uniform
  99. + * address update is found. This is used to make sure that we don't
  100. + * read uniforms when the address is undefined.
  101. + */
  102. + bool needs_uniform_address_update;
  103. +
  104. + /* Set when we find a backwards branch. If the branch is backwards,
  105. + * the taraget is probably doing an address reset to read uniforms,
  106. + * and so we need to be sure that a uniforms address is present in the
  107. + * stream, even if the shader didn't need to read uniforms in later
  108. + * basic blocks.
  109. + */
  110. + bool needs_uniform_address_for_loop;
  111. };
  112. static uint32_t
  113. @@ -227,8 +244,14 @@ check_tmu_write(struct vc4_validated_sha
  114. /* Since direct uses a RADDR uniform reference, it will get counted in
  115. * check_instruction_reads()
  116. */
  117. - if (!is_direct)
  118. + if (!is_direct) {
  119. + if (validation_state->needs_uniform_address_update) {
  120. + DRM_ERROR("Texturing with undefined uniform address\n");
  121. + return false;
  122. + }
  123. +
  124. validated_shader->uniforms_size += 4;
  125. + }
  126. if (submit) {
  127. if (!record_texture_sample(validated_shader,
  128. @@ -242,6 +265,98 @@ check_tmu_write(struct vc4_validated_sha
  129. return true;
  130. }
  131. +static bool require_uniform_address_uniform(struct vc4_validated_shader_info *validated_shader)
  132. +{
  133. + uint32_t o = validated_shader->num_uniform_addr_offsets;
  134. + uint32_t num_uniforms = validated_shader->uniforms_size / 4;
  135. +
  136. + validated_shader->uniform_addr_offsets =
  137. + krealloc(validated_shader->uniform_addr_offsets,
  138. + (o + 1) *
  139. + sizeof(*validated_shader->uniform_addr_offsets),
  140. + GFP_KERNEL);
  141. + if (!validated_shader->uniform_addr_offsets)
  142. + return false;
  143. +
  144. + validated_shader->uniform_addr_offsets[o] = num_uniforms;
  145. + validated_shader->num_uniform_addr_offsets++;
  146. +
  147. + return true;
  148. +}
  149. +
  150. +static bool
  151. +validate_uniform_address_write(struct vc4_validated_shader_info *validated_shader,
  152. + struct vc4_shader_validation_state *validation_state,
  153. + bool is_mul)
  154. +{
  155. + uint64_t inst = validation_state->shader[validation_state->ip];
  156. + u32 add_b = QPU_GET_FIELD(inst, QPU_ADD_B);
  157. + u32 raddr_a = QPU_GET_FIELD(inst, QPU_RADDR_A);
  158. + u32 raddr_b = QPU_GET_FIELD(inst, QPU_RADDR_B);
  159. + u32 add_lri = raddr_add_a_to_live_reg_index(inst);
  160. + /* We want our reset to be pointing at whatever uniform follows the
  161. + * uniforms base address.
  162. + */
  163. + u32 expected_offset = validated_shader->uniforms_size + 4;
  164. +
  165. + /* We only support absolute uniform address changes, and we
  166. + * require that they be in the current basic block before any
  167. + * of its uniform reads.
  168. + *
  169. + * One could potentially emit more efficient QPU code, by
  170. + * noticing that (say) an if statement does uniform control
  171. + * flow for all threads and that the if reads the same number
  172. + * of uniforms on each side. However, this scheme is easy to
  173. + * validate so it's all we allow for now.
  174. + */
  175. +
  176. + if (QPU_GET_FIELD(inst, QPU_SIG) != QPU_SIG_NONE) {
  177. + DRM_ERROR("uniforms address change must be "
  178. + "normal math\n");
  179. + return false;
  180. + }
  181. +
  182. + if (is_mul || QPU_GET_FIELD(inst, QPU_OP_ADD) != QPU_A_ADD) {
  183. + DRM_ERROR("Uniform address reset must be an ADD.\n");
  184. + return false;
  185. + }
  186. +
  187. + if (QPU_GET_FIELD(inst, QPU_COND_ADD) != QPU_COND_ALWAYS) {
  188. + DRM_ERROR("Uniform address reset must be unconditional.\n");
  189. + return false;
  190. + }
  191. +
  192. + if (QPU_GET_FIELD(inst, QPU_PACK) != QPU_PACK_A_NOP &&
  193. + !(inst & QPU_PM)) {
  194. + DRM_ERROR("No packing allowed on uniforms reset\n");
  195. + return false;
  196. + }
  197. +
  198. + if (add_lri == -1) {
  199. + DRM_ERROR("First argument of uniform address write must be "
  200. + "an immediate value.\n");
  201. + return false;
  202. + }
  203. +
  204. + if (validation_state->live_immediates[add_lri] != expected_offset) {
  205. + DRM_ERROR("Resetting uniforms with offset %db instead of %db\n",
  206. + validation_state->live_immediates[add_lri],
  207. + expected_offset);
  208. + return false;
  209. + }
  210. +
  211. + if (!(add_b == QPU_MUX_A && raddr_a == QPU_R_UNIF) &&
  212. + !(add_b == QPU_MUX_B && raddr_b == QPU_R_UNIF)) {
  213. + DRM_ERROR("Second argument of uniform address write must be "
  214. + "a uniform.\n");
  215. + return false;
  216. + }
  217. +
  218. + validation_state->needs_uniform_address_update = false;
  219. + validation_state->needs_uniform_address_for_loop = false;
  220. + return require_uniform_address_uniform(validated_shader);
  221. +}
  222. +
  223. static bool
  224. check_reg_write(struct vc4_validated_shader_info *validated_shader,
  225. struct vc4_shader_validation_state *validation_state,
  226. @@ -251,14 +366,37 @@ check_reg_write(struct vc4_validated_sha
  227. uint32_t waddr = (is_mul ?
  228. QPU_GET_FIELD(inst, QPU_WADDR_MUL) :
  229. QPU_GET_FIELD(inst, QPU_WADDR_ADD));
  230. + uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
  231. + bool ws = inst & QPU_WS;
  232. + bool is_b = is_mul ^ ws;
  233. + u32 lri = waddr_to_live_reg_index(waddr, is_b);
  234. +
  235. + if (lri != -1) {
  236. + uint32_t cond_add = QPU_GET_FIELD(inst, QPU_COND_ADD);
  237. + uint32_t cond_mul = QPU_GET_FIELD(inst, QPU_COND_MUL);
  238. +
  239. + if (sig == QPU_SIG_LOAD_IMM &&
  240. + QPU_GET_FIELD(inst, QPU_PACK) == QPU_PACK_A_NOP &&
  241. + ((is_mul && cond_mul == QPU_COND_ALWAYS) ||
  242. + (!is_mul && cond_add == QPU_COND_ALWAYS))) {
  243. + validation_state->live_immediates[lri] =
  244. + QPU_GET_FIELD(inst, QPU_LOAD_IMM);
  245. + } else {
  246. + validation_state->live_immediates[lri] = ~0;
  247. + }
  248. + }
  249. switch (waddr) {
  250. case QPU_W_UNIFORMS_ADDRESS:
  251. - /* XXX: We'll probably need to support this for reladdr, but
  252. - * it's definitely a security-related one.
  253. - */
  254. - DRM_ERROR("uniforms address load unsupported\n");
  255. - return false;
  256. + if (is_b) {
  257. + DRM_ERROR("relative uniforms address change "
  258. + "unsupported\n");
  259. + return false;
  260. + }
  261. +
  262. + return validate_uniform_address_write(validated_shader,
  263. + validation_state,
  264. + is_mul);
  265. case QPU_W_TLB_COLOR_MS:
  266. case QPU_W_TLB_COLOR_ALL:
  267. @@ -406,9 +544,35 @@ check_instruction_writes(struct vc4_vali
  268. }
  269. static bool
  270. -check_instruction_reads(uint64_t inst,
  271. - struct vc4_validated_shader_info *validated_shader)
  272. +check_branch(uint64_t inst,
  273. + struct vc4_validated_shader_info *validated_shader,
  274. + struct vc4_shader_validation_state *validation_state,
  275. + int ip)
  276. {
  277. + int32_t branch_imm = QPU_GET_FIELD(inst, QPU_BRANCH_TARGET);
  278. + uint32_t waddr_add = QPU_GET_FIELD(inst, QPU_WADDR_ADD);
  279. + uint32_t waddr_mul = QPU_GET_FIELD(inst, QPU_WADDR_MUL);
  280. +
  281. + if ((int)branch_imm < 0)
  282. + validation_state->needs_uniform_address_for_loop = true;
  283. +
  284. + /* We don't want to have to worry about validation of this, and
  285. + * there's no need for it.
  286. + */
  287. + if (waddr_add != QPU_W_NOP || waddr_mul != QPU_W_NOP) {
  288. + DRM_ERROR("branch instruction at %d wrote a register.\n",
  289. + validation_state->ip);
  290. + return false;
  291. + }
  292. +
  293. + return true;
  294. +}
  295. +
  296. +static bool
  297. +check_instruction_reads(struct vc4_validated_shader_info *validated_shader,
  298. + struct vc4_shader_validation_state *validation_state)
  299. +{
  300. + uint64_t inst = validation_state->shader[validation_state->ip];
  301. uint32_t raddr_a = QPU_GET_FIELD(inst, QPU_RADDR_A);
  302. uint32_t raddr_b = QPU_GET_FIELD(inst, QPU_RADDR_B);
  303. uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
  304. @@ -420,6 +584,12 @@ check_instruction_reads(uint64_t inst,
  305. * already be OOM.
  306. */
  307. validated_shader->uniforms_size += 4;
  308. +
  309. + if (validation_state->needs_uniform_address_update) {
  310. + DRM_ERROR("Uniform read with undefined uniform "
  311. + "address\n");
  312. + return false;
  313. + }
  314. }
  315. return true;
  316. @@ -516,6 +686,65 @@ vc4_validate_branches(struct vc4_shader_
  317. return true;
  318. }
  319. +/* Resets any known state for the shader, used when we may be branched to from
  320. + * multiple locations in the program (or at shader start).
  321. + */
  322. +static void
  323. +reset_validation_state(struct vc4_shader_validation_state *validation_state)
  324. +{
  325. + int i;
  326. +
  327. + for (i = 0; i < 8; i++)
  328. + validation_state->tmu_setup[i / 4].p_offset[i % 4] = ~0;
  329. +
  330. + for (i = 0; i < LIVE_REG_COUNT; i++) {
  331. + validation_state->live_min_clamp_offsets[i] = ~0;
  332. + validation_state->live_max_clamp_regs[i] = false;
  333. + validation_state->live_immediates[i] = ~0;
  334. + }
  335. +}
  336. +
  337. +static bool
  338. +texturing_in_progress(struct vc4_shader_validation_state *validation_state)
  339. +{
  340. + return (validation_state->tmu_write_count[0] != 0 ||
  341. + validation_state->tmu_write_count[1] != 0);
  342. +}
  343. +
  344. +static bool
  345. +vc4_handle_branch_target(struct vc4_shader_validation_state *validation_state)
  346. +{
  347. + uint32_t ip = validation_state->ip;
  348. +
  349. + if (!test_bit(ip, validation_state->branch_targets))
  350. + return true;
  351. +
  352. + if (texturing_in_progress(validation_state)) {
  353. + DRM_ERROR("Branch target landed during TMU setup\n");
  354. + return false;
  355. + }
  356. +
  357. + /* Reset our live values tracking, since this instruction may have
  358. + * multiple predecessors.
  359. + *
  360. + * One could potentially do analysis to determine that, for
  361. + * example, all predecessors have a live max clamp in the same
  362. + * register, but we don't bother with that.
  363. + */
  364. + reset_validation_state(validation_state);
  365. +
  366. + /* Since we've entered a basic block from potentially multiple
  367. + * predecessors, we need the uniforms address to be updated before any
  368. + * unforms are read. We require that after any branch point, the next
  369. + * uniform to be loaded is a uniform address offset. That uniform's
  370. + * offset will be marked by the uniform address register write
  371. + * validation, or a one-off the end-of-program check.
  372. + */
  373. + validation_state->needs_uniform_address_update = true;
  374. +
  375. + return true;
  376. +}
  377. +
  378. struct vc4_validated_shader_info *
  379. vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
  380. {
  381. @@ -524,16 +753,12 @@ vc4_validate_shader(struct drm_gem_cma_o
  382. uint32_t ip;
  383. struct vc4_validated_shader_info *validated_shader = NULL;
  384. struct vc4_shader_validation_state validation_state;
  385. - int i;
  386. memset(&validation_state, 0, sizeof(validation_state));
  387. validation_state.shader = shader_obj->vaddr;
  388. validation_state.max_ip = shader_obj->base.size / sizeof(uint64_t);
  389. - for (i = 0; i < 8; i++)
  390. - validation_state.tmu_setup[i / 4].p_offset[i % 4] = ~0;
  391. - for (i = 0; i < ARRAY_SIZE(validation_state.live_min_clamp_offsets); i++)
  392. - validation_state.live_min_clamp_offsets[i] = ~0;
  393. + reset_validation_state(&validation_state);
  394. validation_state.branch_targets =
  395. kcalloc(BITS_TO_LONGS(validation_state.max_ip),
  396. @@ -554,6 +779,9 @@ vc4_validate_shader(struct drm_gem_cma_o
  397. validation_state.ip = ip;
  398. + if (!vc4_handle_branch_target(&validation_state))
  399. + goto fail;
  400. +
  401. switch (sig) {
  402. case QPU_SIG_NONE:
  403. case QPU_SIG_WAIT_FOR_SCOREBOARD:
  404. @@ -569,7 +797,8 @@ vc4_validate_shader(struct drm_gem_cma_o
  405. goto fail;
  406. }
  407. - if (!check_instruction_reads(inst, validated_shader))
  408. + if (!check_instruction_reads(validated_shader,
  409. + &validation_state))
  410. goto fail;
  411. if (sig == QPU_SIG_PROG_END) {
  412. @@ -587,6 +816,11 @@ vc4_validate_shader(struct drm_gem_cma_o
  413. }
  414. break;
  415. + case QPU_SIG_BRANCH:
  416. + if (!check_branch(inst, validated_shader,
  417. + &validation_state, ip))
  418. + goto fail;
  419. + break;
  420. default:
  421. DRM_ERROR("Unsupported QPU signal %d at "
  422. "instruction %d\n", sig, ip);
  423. @@ -607,6 +841,21 @@ vc4_validate_shader(struct drm_gem_cma_o
  424. goto fail;
  425. }
  426. + /* If we did a backwards branch and we haven't emitted a uniforms
  427. + * reset since then, we still need the uniforms stream to have the
  428. + * uniforms address available so that the backwards branch can do its
  429. + * uniforms reset.
  430. + *
  431. + * We could potentially prove that the backwards branch doesn't
  432. + * contain any uses of uniforms until program exit, but that doesn't
  433. + * seem to be worth the trouble.
  434. + */
  435. + if (validation_state.needs_uniform_address_for_loop) {
  436. + if (!require_uniform_address_uniform(validated_shader))
  437. + goto fail;
  438. + validated_shader->uniforms_size += 4;
  439. + }
  440. +
  441. /* Again, no chance of integer overflow here because the worst case
  442. * scenario is 8 bytes of uniforms plus handles per 8-byte
  443. * instruction.