/*
 * Mesa 3-D graphics library
 *
 * Copyright (C) 2012-2013 LunarG, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Authors:
 *    Chia-I Wu <olv@lunarg.com>
 */

#include "toy_compiler.h"

#define CG_REG_SHIFT 5
#define CG_REG_NUM(origin) ((origin) >> CG_REG_SHIFT)

struct codegen {
   const struct ilo_dev *dev;
   const struct toy_inst *inst;
   int pc;

   unsigned flag_reg_num;
   unsigned flag_sub_reg_num;

   struct codegen_dst {
      unsigned file;
      unsigned type;
      bool indirect;
      unsigned indirect_subreg;
      unsigned origin; /* (RegNum << 5 | SubRegNumInBytes) */

      unsigned horz_stride;

      unsigned writemask;
   } dst;

   struct codegen_src {
      unsigned file;
      unsigned type;
      bool indirect;
      unsigned indirect_subreg;
      unsigned origin; /* (RegNum << 5 | SubRegNumInBytes) */

      unsigned vert_stride;
      unsigned width;
      unsigned horz_stride;

      unsigned swizzle[4];
      bool absolute;
      bool negate;
   } src[3];
};

/*
 * From the Sandy Bridge PRM, volume 4 part 2, page 107-108:
 *
 *     "(Src0Index) The 5-bit index for source 0. The 12-bit table-look-up
 *      result forms bits [88:77], the source 0 register region fields, of the
 *      128-bit instruction word."
 *
 *     "(SubRegIndex) The 5-bit index for sub-register fields. The 15-bit
 *      table-look-up result forms bits [100:96], [68,64] and [52,48] of the
 *      128-bit instruction word."
 *
 *     "(DataTypeIndex) The 5-bit index for data type fields. The 18-bit
 *      table-look-up result forms bits [63:61] and [46, 32] of the 128-bit
 *      instruction word."
 *
 *     "(ControlIndex) The 5-bit index for data type fields. The 17-bit
 *      table-look-up result forms bits[31], and [23, 8] of the 128-bit
 *      instruction word."
 */
static const struct toy_compaction_table toy_compaction_table_gen6 = {
   .control = {
      [0]   = 0x00000,  /* 00000000000000000 */
      [1]   = 0x08000,  /* 01000000000000000 */
      [2]   = 0x06000,  /* 00110000000000000 */
      [3]   = 0x00100,  /* 00000000100000000 */
      [4]   = 0x02000,  /* 00010000000000000 */
      [5]   = 0x01100,  /* 00001000100000000 */
      [6]   = 0x00102,  /* 00000000100000010 */
      [7]   = 0x00002,  /* 00000000000000010 */
      [8]   = 0x08100,  /* 01000000100000000 */
      [9]   = 0x0a000,  /* 01010000000000000 */
      [10]  = 0x16000,  /* 10110000000000000 */
      [11]  = 0x04000,  /* 00100000000000000 */
      [12]  = 0x1a000,  /* 11010000000000000 */
      [13]  = 0x18000,  /* 11000000000000000 */
      [14]  = 0x09100,  /* 01001000100000000 */
      [15]  = 0x08008,  /* 01000000000001000 */
      [16]  = 0x08004,  /* 01000000000000100 */
      [17]  = 0x00008,  /* 00000000000001000 */
      [18]  = 0x00004,  /* 00000000000000100 */
      [19]  = 0x01100,  /* 00111000100000000 */
      [20]  = 0x01102,  /* 00001000100000010 */
      [21]  = 0x06100,  /* 00110000100000000 */
      [22]  = 0x06001,  /* 00110000000000001 */
      [23]  = 0x04001,  /* 00100000000000001 */
      [24]  = 0x06002,  /* 00110000000000010 */
      [25]  = 0x06005,  /* 00110000000000101 */
      [26]  = 0x06009,  /* 00110000000001001 */
      [27]  = 0x06010,  /* 00110000000010000 */
      [28]  = 0x06003,  /* 00110000000000011 */
      [29]  = 0x06004,  /* 00110000000000100 */
      [30]  = 0x06108,  /* 00110000100001000 */
      [31]  = 0x04009,  /* 00100000000001001 */
   },
   .datatype = {
      [0]   = 0x09c00,  /* 001001110000000000 */
      [1]   = 0x08c20,  /* 001000110000100000 */
      [2]   = 0x09c01,  /* 001001110000000001 */
      [3]   = 0x08060,  /* 001000000001100000 */
      [4]   = 0x0ad29,  /* 001010110100101001 */
      [5]   = 0x081ad,  /* 001000000110101101 */
      [6]   = 0x0c62c,  /* 001100011000101100 */
      [7]   = 0x0bdad,  /* 001011110110101101 */
      [8]   = 0x081ec,  /* 001000000111101100 */
      [9]   = 0x08061,  /* 001000000001100001 */
      [10]  = 0x08ca5,  /* 001000110010100101 */
      [11]  = 0x08041,  /* 001000000001000001 */
      [12]  = 0x08231,  /* 001000001000110001 */
      [13]  = 0x08229,  /* 001000001000101001 */
      [14]  = 0x08020,  /* 001000000000100000 */
      [15]  = 0x08232,  /* 001000001000110010 */
      [16]  = 0x0a529,  /* 001010010100101001 */
      [17]  = 0x0b4a5,  /* 001011010010100101 */
      [18]  = 0x081a5,  /* 001000000110100101 */
      [19]  = 0x0c629,  /* 001100011000101001 */
      [20]  = 0x0b62c,  /* 001011011000101100 */
      [21]  = 0x0b5a5,  /* 001011010110100101 */
      [22]  = 0x0bda5,  /* 001011110110100101 */
      [23]  = 0x0f1bd,  /* 001111011110111101 */
      [24]  = 0x0f1bc,  /* 001111011110111100 */
      [25]  = 0x0f1bd,  /* 001111011110111101 */
      [26]  = 0x0f19d,  /* 001111011110011101 */
      [27]  = 0x0f1be,  /* 001111011110111110 */
      [28]  = 0x08021,  /* 001000000000100001 */
      [29]  = 0x08022,  /* 001000000000100010 */
      [30]  = 0x09fdd,  /* 001001111111011101 */
      [31]  = 0x083be,  /* 001000001110111110 */
   },
   .subreg = {
      [0]   = 0x0000,   /* 000000000000000 */
      [1]   = 0x0004,   /* 000000000000100 */
      [2]   = 0x0180,   /* 000000110000000 */
      [3]   = 0x1000,   /* 111000000000000 */
      [4]   = 0x3c08,   /* 011110000001000 */
      [5]   = 0x0400,   /* 000010000000000 */
      [6]   = 0x0010,   /* 000000000010000 */
      [7]   = 0x0c0c,   /* 000110000001100 */
      [8]   = 0x1000,   /* 001000000000000 */
      [9]   = 0x0200,   /* 000001000000000 */
      [10]  = 0x0294,   /* 000001010010100 */
      [11]  = 0x0056,   /* 000000001010110 */
      [12]  = 0x2000,   /* 010000000000000 */
      [13]  = 0x6000,   /* 110000000000000 */
      [14]  = 0x0800,   /* 000100000000000 */
      [15]  = 0x0080,   /* 000000010000000 */
      [16]  = 0x0008,   /* 000000000001000 */
      [17]  = 0x4000,   /* 100000000000000 */
      [18]  = 0x0280,   /* 000001010000000 */
      [19]  = 0x1400,   /* 001010000000000 */
      [20]  = 0x1800,   /* 001100000000000 */
      [21]  = 0x0054,   /* 000000001010100 */
      [22]  = 0x5a94,   /* 101101010010100 */
      [23]  = 0x2800,   /* 010100000000000 */
      [24]  = 0x008f,   /* 000000010001111 */
      [25]  = 0x3000,   /* 011000000000000 */
      [26]  = 0x1c00,   /* 111110000000000 */
      [27]  = 0x5000,   /* 101000000000000 */
      [28]  = 0x000f,   /* 000000000001111 */
      [29]  = 0x088f,   /* 000100010001111 */
      [30]  = 0x108f,   /* 001000010001111 */
      [31]  = 0x0c00,   /* 000110000000000 */
   },
   .src = {
      [0]   = 0x000,    /* 000000000000 */
      [1]   = 0x588,    /* 010110001000 */
      [2]   = 0x468,    /* 010001101000 */
      [3]   = 0x228,    /* 001000101000 */
      [4]   = 0x690,    /* 011010010000 */
      [5]   = 0x120,    /* 000100100000 */
      [6]   = 0x46c,    /* 010001101100 */
      [7]   = 0x510,    /* 010101110000 */
      [8]   = 0x618,    /* 011001111000 */
      [9]   = 0x328,    /* 001100101000 */
      [10]  = 0x58c,    /* 010110001100 */
      [11]  = 0x220,    /* 001000100000 */
      [12]  = 0x58a,    /* 010110001010 */
      [13]  = 0x002,    /* 000000000010 */
      [14]  = 0x550,    /* 010101010000 */
      [15]  = 0x568,    /* 010101101000 */
      [16]  = 0xf4c,    /* 111101001100 */
      [17]  = 0xf2c,    /* 111100101100 */
      [18]  = 0x610,    /* 011001110000 */
      [19]  = 0x589,    /* 010110001001 */
      [20]  = 0x558,    /* 010101011000 */
      [21]  = 0x348,    /* 001101001000 */
      [22]  = 0x42c,    /* 010000101100 */
      [23]  = 0x400,    /* 010000000000 */
      [24]  = 0x310,    /* 001101110000 */
      [25]  = 0x310,    /* 001100010000 */
      [26]  = 0x300,    /* 001100000000 */
      [27]  = 0x46a,    /* 010001101010 */
      [28]  = 0x318,    /* 001101111000 */
      [29]  = 0x010,    /* 000001110000 */
      [30]  = 0x320,    /* 001100100000 */
      [31]  = 0x350,    /* 001101010000 */
   },
};

/*
 * From the Ivy Bridge PRM, volume 4 part 3, page 128:
 *
 *     "(Src0Index) Lookup one of 32 12-bit values. That value is used (from
 *      MSB to LSB) for the Src0.AddrMode, Src0.ChanSel[7:4], Src0.HorzStride,
 *      Src0.SrcMod, Src0.VertStride, and Src0.Width bit fields."
 *
 *     "(SubRegIndex) Lookup one of 32 15-bit values. That value is used (from
 *      MSB to LSB) for various fields for Src1, Src0, and Dst, including
 *      ChanEn/ChanSel, SubRegNum, and AddrImm[4] or AddrImm[4:0], depending
 *      on AddrMode and AccessMode.
 *
 *     "(DataTypeIndex) Lookup one of 32 18-bit values. That value is used
 *      (from MSB to LSB) for the Dst.AddrMode, Dst.HorzStride, Dst.DstType,
 *      Dst.RegFile, Src0.SrcType, Src0.RegFile, Src1.SrcType, and
 *      Src1.RegType bit fields."
 *
 *     "(ControlIndex) Lookup one of 32 19-bit values. That value is used
 *      (from MSB to LSB) for the FlagRegNum, FlagSubRegNum, Saturate,
 *      ExecSize, PredInv, PredCtrl, ThreadCtrl, QtrCtrl, DepCtrl, MaskCtrl,
 *      and AccessMode bit fields."
 */
static const struct toy_compaction_table toy_compaction_table_gen7 = {
   .control = {
      [0]   = 0x00002,  /* 0000000000000000010 */
      [1]   = 0x04000,  /* 0000100000000000000 */
      [2]   = 0x04001,  /* 0000100000000000001 */
      [3]   = 0x04002,  /* 0000100000000000010 */
      [4]   = 0x04003,  /* 0000100000000000011 */
      [5]   = 0x04004,  /* 0000100000000000100 */
      [6]   = 0x04005,  /* 0000100000000000101 */
      [7]   = 0x04007,  /* 0000100000000000111 */
      [8]   = 0x04008,  /* 0000100000000001000 */
      [9]   = 0x04009,  /* 0000100000000001001 */
      [10]  = 0x0400d,  /* 0000100000000001101 */
      [11]  = 0x06000,  /* 0000110000000000000 */
      [12]  = 0x06001,  /* 0000110000000000001 */
      [13]  = 0x06002,  /* 0000110000000000010 */
      [14]  = 0x06003,  /* 0000110000000000011 */
      [15]  = 0x06004,  /* 0000110000000000100 */
      [16]  = 0x06005,  /* 0000110000000000101 */
      [17]  = 0x06007,  /* 0000110000000000111 */
      [18]  = 0x06009,  /* 0000110000000001001 */
      [19]  = 0x0600d,  /* 0000110000000001101 */
      [20]  = 0x06010,  /* 0000110000000010000 */
      [21]  = 0x06100,  /* 0000110000100000000 */
      [22]  = 0x08000,  /* 0001000000000000000 */
      [23]  = 0x08002,  /* 0001000000000000010 */
      [24]  = 0x08004,  /* 0001000000000000100 */
      [25]  = 0x08100,  /* 0001000000100000000 */
      [26]  = 0x16000,  /* 0010110000000000000 */
      [27]  = 0x16010,  /* 0010110000000010000 */
      [28]  = 0x18000,  /* 0011000000000000000 */
      [29]  = 0x18100,  /* 0011000000100000000 */
      [30]  = 0x28000,  /* 0101000000000000000 */
      [31]  = 0x28100,  /* 0101000000100000000 */
   },
   .datatype = {
      [0]   = 0x08001,  /* 001000000000000001 */
      [1]   = 0x08020,  /* 001000000000100000 */
      [2]   = 0x08021,  /* 001000000000100001 */
      [3]   = 0x08061,  /* 001000000001100001 */
      [4]   = 0x080bd,  /* 001000000010111101 */
      [5]   = 0x082fd,  /* 001000001011111101 */
      [6]   = 0x083a1,  /* 001000001110100001 */
      [7]   = 0x083a5,  /* 001000001110100101 */
      [8]   = 0x083bd,  /* 001000001110111101 */
      [9]   = 0x08421,  /* 001000010000100001 */
      [10]  = 0x08c20,  /* 001000110000100000 */
      [11]  = 0x08c21,  /* 001000110000100001 */
      [12]  = 0x094a5,  /* 001001010010100101 */
      [13]  = 0x09ca4,  /* 001001110010100100 */
      [14]  = 0x09ca5,  /* 001001110010100101 */
      [15]  = 0x0f3bd,  /* 001111001110111101 */
      [16]  = 0x0f79d,  /* 001111011110011101 */
      [17]  = 0x0f7bc,  /* 001111011110111100 */
      [18]  = 0x0f7bd,  /* 001111011110111101 */
      [19]  = 0x0ffbc,  /* 001111111110111100 */
      [20]  = 0x0020c,  /* 000000001000001100 */
      [21]  = 0x0803d,  /* 001000000000111101 */
      [22]  = 0x080a5,  /* 001000000010100101 */
      [23]  = 0x08420,  /* 001000010000100000 */
      [24]  = 0x094a4,  /* 001001010010100100 */
      [25]  = 0x09c84,  /* 001001110010000100 */
      [26]  = 0x0a509,  /* 001010010100001001 */
      [27]  = 0x0dfbd,  /* 001101111110111101 */
      [28]  = 0x0ffbd,  /* 001111111110111101 */
      [29]  = 0x0bdac,  /* 001011110110101100 */
      [30]  = 0x0a528,  /* 001010010100101000 */
      [31]  = 0x0ad28,  /* 001010110100101000 */
   },
   .subreg = {
      [0]   = 0x0000,   /* 000000000000000 */
      [1]   = 0x0001,   /* 000000000000001 */
      [2]   = 0x0008,   /* 000000000001000 */
      [3]   = 0x000f,   /* 000000000001111 */
      [4]   = 0x0010,   /* 000000000010000 */
      [5]   = 0x0080,   /* 000000010000000 */
      [6]   = 0x0100,   /* 000000100000000 */
      [7]   = 0x0180,   /* 000000110000000 */
      [8]   = 0x0200,   /* 000001000000000 */
      [9]   = 0x0210,   /* 000001000010000 */
      [10]  = 0x0280,   /* 000001010000000 */
      [11]  = 0x1000,   /* 001000000000000 */
      [12]  = 0x1001,   /* 001000000000001 */
      [13]  = 0x1081,   /* 001000010000001 */
      [14]  = 0x1082,   /* 001000010000010 */
      [15]  = 0x1083,   /* 001000010000011 */
      [16]  = 0x1084,   /* 001000010000100 */
      [17]  = 0x1087,   /* 001000010000111 */
      [18]  = 0x1088,   /* 001000010001000 */
      [19]  = 0x108e,   /* 001000010001110 */
      [20]  = 0x108f,   /* 001000010001111 */
      [21]  = 0x1180,   /* 001000110000000 */
      [22]  = 0x11e8,   /* 001000111101000 */
      [23]  = 0x2000,   /* 010000000000000 */
      [24]  = 0x2180,   /* 010000110000000 */
      [25]  = 0x3000,   /* 011000000000000 */
      [26]  = 0x3c87,   /* 011110010000111 */
      [27]  = 0x4000,   /* 100000000000000 */
      [28]  = 0x5000,   /* 101000000000000 */
      [29]  = 0x6000,   /* 110000000000000 */
      [30]  = 0x7000,   /* 111000000000000 */
      [31]  = 0x701c,   /* 111000000011100 */
   },
   .src = {
      [0]   = 0x000,    /* 000000000000 */
      [1]   = 0x002,    /* 000000000010 */
      [2]   = 0x010,    /* 000000010000 */
      [3]   = 0x012,    /* 000000010010 */
      [4]   = 0x018,    /* 000000011000 */
      [5]   = 0x020,    /* 000000100000 */
      [6]   = 0x028,    /* 000000101000 */
      [7]   = 0x048,    /* 000001001000 */
      [8]   = 0x050,    /* 000001010000 */
      [9]   = 0x070,    /* 000001110000 */
      [10]  = 0x078,    /* 000001111000 */
      [11]  = 0x300,    /* 001100000000 */
      [12]  = 0x302,    /* 001100000010 */
      [13]  = 0x308,    /* 001100001000 */
      [14]  = 0x310,    /* 001100010000 */
      [15]  = 0x312,    /* 001100010010 */
      [16]  = 0x320,    /* 001100100000 */
      [17]  = 0x328,    /* 001100101000 */
      [18]  = 0x338,    /* 001100111000 */
      [19]  = 0x340,    /* 001101000000 */
      [20]  = 0x342,    /* 001101000010 */
      [21]  = 0x348,    /* 001101001000 */
      [22]  = 0x350,    /* 001101010000 */
      [23]  = 0x360,    /* 001101100000 */
      [24]  = 0x368,    /* 001101101000 */
      [25]  = 0x370,    /* 001101110000 */
      [26]  = 0x371,    /* 001101110001 */
      [27]  = 0x378,    /* 001101111000 */
      [28]  = 0x468,    /* 010001101000 */
      [29]  = 0x469,    /* 010001101001 */
      [30]  = 0x46a,    /* 010001101010 */
      [31]  = 0x588,    /* 010110001000 */
   },
};

static const struct toy_compaction_table toy_compaction_table_gen8 = {
   .control = {
   },
   .datatype = {
   },
   .subreg = {
   },
   .src = {
   },
   .control_3src = {
   },
   .source_3src = {
   },
};

const struct toy_compaction_table *
toy_compiler_get_compaction_table(const struct ilo_dev *dev)
{
   switch (ilo_dev_gen(dev)) {
   case ILO_GEN(8):
      return &toy_compaction_table_gen8;
   case ILO_GEN(7.5):
   case ILO_GEN(7):
      return &toy_compaction_table_gen7;
   case ILO_GEN(6):
      return &toy_compaction_table_gen6;
   default:
      assert(!"unsupported gen");
      return NULL;
   }
}

/**
 * Return true if the source operand is null.
 */
static bool
src_is_null(const struct codegen *cg, int idx)
{
   const struct codegen_src *src = &cg->src[idx];

   return (src->file == GEN6_FILE_ARF &&
           src->origin == GEN6_ARF_NULL << CG_REG_SHIFT);
}

/**
 * Translate a source operand to DW2 or DW3 of the 1-src/2-src format.
 */
static uint32_t
translate_src_gen6(const struct codegen *cg, int idx)
{
   const struct codegen_src *src = &cg->src[idx];
   uint32_t dw;

   ILO_DEV_ASSERT(cg->dev, 6, 8);

   /* special treatment may be needed if any of the operand is immediate */
   if (cg->src[0].file == GEN6_FILE_IMM) {
      assert(!cg->src[0].absolute && !cg->src[0].negate);

      /* only the last src operand can be an immediate unless it is Gen8+ */
      assert(ilo_dev_gen(cg->dev) >= ILO_GEN(8) || src_is_null(cg, 1));

      if (!src_is_null(cg, 1))
         return cg->src[idx].origin;

      if (idx == 0) {
         if (ilo_dev_gen(cg->dev) >= ILO_GEN(8)) {
            return cg->src[1].type << 27 |
                   cg->src[1].file << 25;
         } else {
            return cg->flag_sub_reg_num << 25;
         }
      } else {
         return cg->src[0].origin;
      }
   }
   else if (idx && cg->src[1].file == GEN6_FILE_IMM) {
      assert(!cg->src[1].absolute && !cg->src[1].negate);
      return cg->src[1].origin;
   }

   assert(src->file != GEN6_FILE_IMM);

   if (src->indirect) {
      const int offset = (int) src->origin;

      assert(src->file == GEN6_FILE_GRF);
      assert(offset < 512 && offset >= -512);

      if (cg->inst->access_mode == GEN6_ALIGN_16) {
         assert(src->width == GEN6_WIDTH_4);
         assert(src->horz_stride == GEN6_HORZSTRIDE_1);

         /* the lower 4 bits are reserved for the swizzle_[xy] */
         assert(!(src->origin & 0xf));

         dw = src->vert_stride << 21 |
              src->swizzle[3] << 18 |
              src->swizzle[2] << 16 |
              GEN6_ADDRMODE_INDIRECT << 15 |
              src->negate << 14 |
              src->absolute << 13 |
              src->swizzle[1] << 2 |
              src->swizzle[0];
         if (ilo_dev_gen(cg->dev) >= ILO_GEN(8)) {
            dw |= src->indirect_subreg << 9 |
                  (src->origin & 0x1f0);
         } else {
            dw |= src->indirect_subreg << 10 |
                  (src->origin & 0x3f0);
         }
      }
      else {
         assert(src->swizzle[0] == TOY_SWIZZLE_X &&
                src->swizzle[1] == TOY_SWIZZLE_Y &&
                src->swizzle[2] == TOY_SWIZZLE_Z &&
                src->swizzle[3] == TOY_SWIZZLE_W);

         dw = src->vert_stride << 21 |
              src->width << 18 |
              src->horz_stride << 16 |
              GEN6_ADDRMODE_INDIRECT << 15 |
              src->negate << 14 |
              src->absolute << 13;
         if (ilo_dev_gen(cg->dev) >= ILO_GEN(8)) {
            dw |= src->indirect_subreg << 9 |
                  (src->origin & 0x1ff);
         } else {
            dw |= src->indirect_subreg << 10 |
                  (src->origin & 0x3ff);
         }
      }
   }
   else {
      switch (src->file) {
      case GEN6_FILE_ARF:
         break;
      case GEN6_FILE_GRF:
         assert(CG_REG_NUM(src->origin) < 128);
         break;
      case GEN6_FILE_MRF:
         assert(cg->inst->opcode == GEN6_OPCODE_SEND ||
                cg->inst->opcode == GEN6_OPCODE_SENDC);
         assert(CG_REG_NUM(src->origin) < 16);
         break;
      case GEN6_FILE_IMM:
      default:
         assert(!"invalid src file");
         break;
      }

      if (cg->inst->access_mode == GEN6_ALIGN_16) {
         assert(src->width == GEN6_WIDTH_4);
         assert(src->horz_stride == GEN6_HORZSTRIDE_1);

         /* the lower 4 bits are reserved for the swizzle_[xy] */
         assert(!(src->origin & 0xf));

         dw = src->vert_stride << 21 |
              src->swizzle[3] << 18 |
              src->swizzle[2] << 16 |
              GEN6_ADDRMODE_DIRECT << 15 |
              src->negate << 14 |
              src->absolute << 13 |
              src->origin |
              src->swizzle[1] << 2 |
              src->swizzle[0];
      }
      else {
         assert(src->swizzle[0] == TOY_SWIZZLE_X &&
                src->swizzle[1] == TOY_SWIZZLE_Y &&
                src->swizzle[2] == TOY_SWIZZLE_Z &&
                src->swizzle[3] == TOY_SWIZZLE_W);

         dw = src->vert_stride << 21 |
              src->width << 18 |
              src->horz_stride << 16 |
              GEN6_ADDRMODE_DIRECT << 15 |
              src->negate << 14 |
              src->absolute << 13 |
              src->origin;
      }
   }

   if (ilo_dev_gen(cg->dev) >= ILO_GEN(8)) {
      const bool indirect_origin_bit9 = (cg->dst.indirect) ?
         (src->origin & 0x200) : 0;

      if (idx == 0) {
         dw |= indirect_origin_bit9 << 31 |
               cg->src[1].type << 27 |
               cg->src[1].file << 25;
      } else {
         dw |= indirect_origin_bit9 << 25;
      }
   } else {
      if (idx == 0)
         dw |= cg->flag_sub_reg_num << 25;
   }

   return dw;
}

/**
 * Translate the destination operand to the higher 16 bits of DW1 of the
 * 1-src/2-src format.
 */
static uint16_t
translate_dst_region_gen6(const struct codegen *cg)
{
   const struct codegen_dst *dst = &cg->dst;
   uint16_t dw1_region;

   ILO_DEV_ASSERT(cg->dev, 6, 8);

   if (dst->file == GEN6_FILE_IMM) {
      /* dst is immediate (JIP) when the opcode is a conditional branch */
      switch (cg->inst->opcode) {
      case GEN6_OPCODE_IF:
      case GEN6_OPCODE_ELSE:
      case GEN6_OPCODE_ENDIF:
      case GEN6_OPCODE_WHILE:
         assert(dst->type == GEN6_TYPE_W);
         dw1_region = (dst->origin & 0xffff);
         break;
      default:
         assert(!"dst cannot be immediate");
         dw1_region = 0;
         break;
      }

      return dw1_region;
   }

   if (dst->indirect) {
      const int offset = (int) dst->origin;

      assert(dst->file == GEN6_FILE_GRF);
      assert(offset < 512 && offset >= -512);

      if (cg->inst->access_mode == GEN6_ALIGN_16) {
         /*
          * From the Sandy Bridge PRM, volume 4 part 2, page 144:
          *
          *     "Allthough Dst.HorzStride is a don't care for Align16, HW
          *      needs this to be programmed as 01."
          */
         assert(dst->horz_stride == GEN6_HORZSTRIDE_1);
         /* the lower 4 bits are reserved for the writemask */
         assert(!(dst->origin & 0xf));

         dw1_region = GEN6_ADDRMODE_INDIRECT << 15 |
                      dst->horz_stride << 13 |
                      dst->writemask;
         if (ilo_dev_gen(cg->dev) >= ILO_GEN(8)) {
            dw1_region |= dst->indirect_subreg << 9 |
                          (dst->origin & 0x1f0);
         } else {
            dw1_region |= dst->indirect_subreg << 10 |
                          (dst->origin & 0x3f0);
         }
      }
      else {
         assert(dst->writemask == TOY_WRITEMASK_XYZW);

         dw1_region = GEN6_ADDRMODE_INDIRECT << 15 |
                      dst->horz_stride << 13;
         if (ilo_dev_gen(cg->dev) >= ILO_GEN(8)) {
            dw1_region |= dst->indirect_subreg << 9 |
                          (dst->origin & 0x1ff);
         } else {
            dw1_region |= dst->indirect_subreg << 10 |
                          (dst->origin & 0x3ff);
         }
      }
   }
   else {
      assert((dst->file == GEN6_FILE_GRF &&
              CG_REG_NUM(dst->origin) < 128) ||
             (dst->file == GEN6_FILE_MRF &&
              CG_REG_NUM(dst->origin) < 16) ||
             (dst->file == GEN6_FILE_ARF));

      if (cg->inst->access_mode == GEN6_ALIGN_16) {
         /* similar to the indirect case */
         assert(dst->horz_stride == GEN6_HORZSTRIDE_1);
         assert(!(dst->origin & 0xf));

         dw1_region = GEN6_ADDRMODE_DIRECT << 15 |
                      dst->horz_stride << 13 |
                      dst->origin |
                      dst->writemask;
      }
      else {
         assert(dst->writemask == TOY_WRITEMASK_XYZW);

         dw1_region = GEN6_ADDRMODE_DIRECT << 15 |
                      dst->horz_stride << 13 |
                      dst->origin;
      }
   }

   return dw1_region;
}

/**
 * Translate the destination operand to DW1 of the 1-src/2-src format.
 */
static uint32_t
translate_dst_gen6(const struct codegen *cg)
{
   ILO_DEV_ASSERT(cg->dev, 6, 7.5);

   return translate_dst_region_gen6(cg) << 16 |
          cg->src[1].type << 12 |
          cg->src[1].file << 10 |
          cg->src[0].type << 7 |
          cg->src[0].file << 5 |
          cg->dst.type << 2 |
          cg->dst.file;
}

static uint32_t
translate_dst_gen8(const struct codegen *cg)
{
   const bool indirect_origin_bit9 = (cg->dst.indirect) ?
      (cg->dst.origin & 0x200) : 0;

   ILO_DEV_ASSERT(cg->dev, 8, 8);

   return translate_dst_region_gen6(cg) << 16 |
          indirect_origin_bit9 << 15 |
          cg->src[0].type << 11 |
          cg->src[0].file << 9 |
          cg->dst.type << 5 |
          cg->dst.file << 3 |
          cg->inst->mask_ctrl << 2 |
          cg->flag_reg_num << 1 |
          cg->flag_sub_reg_num;
}

/**
 * Translate the instruction to DW0 of the 1-src/2-src format.
 */
static uint32_t
translate_inst_gen6(const struct codegen *cg)
{
   const bool debug_ctrl = false;
   const bool cmpt_ctrl = false;

   ILO_DEV_ASSERT(cg->dev, 6, 7.5);

   assert(cg->inst->opcode < 128);

   return cg->inst->saturate << 31 |
          debug_ctrl << 30 |
          cmpt_ctrl << 29 |
          cg->inst->acc_wr_ctrl << 28 |
          cg->inst->cond_modifier << 24 |
          cg->inst->exec_size << 21 |
          cg->inst->pred_inv << 20 |
          cg->inst->pred_ctrl << 16 |
          cg->inst->thread_ctrl << 14 |
          cg->inst->qtr_ctrl << 12 |
          cg->inst->dep_ctrl << 10 |
          cg->inst->mask_ctrl << 9 |
          cg->inst->access_mode << 8 |
          cg->inst->opcode;
}

static uint32_t
translate_inst_gen8(const struct codegen *cg)
{
   const bool debug_ctrl = false;
   const bool cmpt_ctrl = false;

   ILO_DEV_ASSERT(cg->dev, 8, 8);

   assert(cg->inst->opcode < 128);

   return cg->inst->saturate << 31 |
          debug_ctrl << 30 |
          cmpt_ctrl << 29 |
          cg->inst->acc_wr_ctrl << 28 |
          cg->inst->cond_modifier << 24 |
          cg->inst->exec_size << 21 |
          cg->inst->pred_inv << 20 |
          cg->inst->pred_ctrl << 16 |
          cg->inst->thread_ctrl << 14 |
          cg->inst->qtr_ctrl << 12 |
          cg->inst->dep_ctrl << 9 |
          cg->inst->access_mode << 8 |
          cg->inst->opcode;
}

/**
 * Codegen an instruction in 1-src/2-src format.
 */
static void
codegen_inst_gen6(const struct codegen *cg, uint32_t *code)
{
   ILO_DEV_ASSERT(cg->dev, 6, 8);

   if (ilo_dev_gen(cg->dev) >= ILO_GEN(8)) {
      code[0] = translate_inst_gen8(cg);
      code[1] = translate_dst_gen8(cg);
   } else {
      code[0] = translate_inst_gen6(cg);
      code[1] = translate_dst_gen6(cg);
   }

   code[2] = translate_src_gen6(cg, 0);
   code[3] = translate_src_gen6(cg, 1);
   assert(src_is_null(cg, 2));
}

/**
 * Codegen an instruction in 3-src format.
 */
static void
codegen_inst_3src_gen6(const struct codegen *cg, uint32_t *code)
{
   const struct codegen_dst *dst = &cg->dst;
   uint32_t dw0, dw1, dw_src[3];
   int i;

   ILO_DEV_ASSERT(cg->dev, 6, 8);

   if (ilo_dev_gen(cg->dev) >= ILO_GEN(8))
      dw0 = translate_inst_gen8(cg);
   else
      dw0 = translate_inst_gen6(cg);

   /*
    * 3-src instruction restrictions
    *
    *  - align16 with direct addressing
    *  - GRF or MRF dst
    *  - GRF src
    *  - sub_reg_num is DWORD aligned
    *  - no regioning except replication control
    *    (vert_stride == 0 && horz_stride == 0)
    */
   assert(cg->inst->access_mode == GEN6_ALIGN_16);

   assert(!dst->indirect);
   assert((dst->file == GEN6_FILE_GRF && CG_REG_NUM(dst->origin) < 128) ||
          (dst->file == GEN6_FILE_MRF && CG_REG_NUM(dst->origin) < 16));
   assert(!(dst->origin & 0x3));
   assert(dst->horz_stride == GEN6_HORZSTRIDE_1);

   if (ilo_dev_gen(cg->dev) >= ILO_GEN(8)) {
      dw1 = dst->origin << 19 |
            dst->writemask << 17 |
            cg->src[2].negate << 10 |
            cg->src[2].negate << 10 |
            cg->src[2].absolute << 9 |
            cg->src[1].negate << 8 |
            cg->src[1].absolute << 7 |
            cg->src[0].negate << 6 |
            cg->src[0].absolute << 5 |
            cg->inst->mask_ctrl << 2 |
            cg->flag_reg_num << 1 |
            cg->flag_sub_reg_num;
   } else {
      dw1 = dst->origin << 19 |
            dst->writemask << 17 |
            cg->src[2].negate << 9 |
            cg->src[2].absolute << 8 |
            cg->src[1].negate << 7 |
            cg->src[1].absolute << 6 |
            cg->src[0].negate << 5 |
            cg->src[0].absolute << 4 |
            cg->flag_sub_reg_num << 1 |
            (dst->file == GEN6_FILE_MRF);
   }

   for (i = 0; i < 3; i++) {
      const struct codegen_src *src = &cg->src[i];

      assert(!src->indirect);
      assert(src->file == GEN6_FILE_GRF && CG_REG_NUM(src->origin) < 128);
      assert(!(src->origin & 0x3));

      assert((src->vert_stride == GEN6_VERTSTRIDE_4 &&
              src->horz_stride == GEN6_HORZSTRIDE_1) ||
             (src->vert_stride == GEN6_VERTSTRIDE_0 &&
              src->horz_stride == GEN6_HORZSTRIDE_0));
      assert(src->width == GEN6_WIDTH_4);

      dw_src[i] = src->origin << 7 |
                  src->swizzle[3] << 7 |
                  src->swizzle[2] << 5 |
                  src->swizzle[1] << 3 |
                  src->swizzle[0] << 1 |
                  (src->vert_stride == GEN6_VERTSTRIDE_0 &&
                   src->horz_stride == GEN6_HORZSTRIDE_0);

      /* only the lower 20 bits are used */
      assert((dw_src[i] & 0xfffff) == dw_src[i]);
   }

   code[0] = dw0;
   code[1] = dw1;
   /* concatenate the bits of dw_src */
   code[2] = (dw_src[1] & 0x7ff ) << 21 | dw_src[0];
   code[3] = dw_src[2] << 10 | (dw_src[1] >> 11);
}

/**
 * Sanity check the region parameters of the operands.
 */
static void
codegen_validate_region_restrictions(const struct codegen *cg)
{
   const int exec_size_map[] = {
      [GEN6_EXECSIZE_1] = 1,
      [GEN6_EXECSIZE_2] = 2,
      [GEN6_EXECSIZE_4] = 4,
      [GEN6_EXECSIZE_8] = 8,
      [GEN6_EXECSIZE_16] = 16,
      [GEN6_EXECSIZE_32] = 32,
   };
   const int width_map[] = {
      [GEN6_WIDTH_1] = 1,
      [GEN6_WIDTH_2] = 2,
      [GEN6_WIDTH_4] = 4,
      [GEN6_WIDTH_8] = 8,
      [GEN6_WIDTH_16] = 16,
   };
   const int horz_stride_map[] = {
      [GEN6_HORZSTRIDE_0] = 0,
      [GEN6_HORZSTRIDE_1] = 1,
      [GEN6_HORZSTRIDE_2] = 2,
      [GEN6_HORZSTRIDE_4] = 4,
   };
   const int vert_stride_map[] = {
      [GEN6_VERTSTRIDE_0] = 0,
      [GEN6_VERTSTRIDE_1] = 1,
      [GEN6_VERTSTRIDE_2] = 2,
      [GEN6_VERTSTRIDE_4] = 4,
      [GEN6_VERTSTRIDE_8] = 8,
      [GEN6_VERTSTRIDE_16] = 16,
      [GEN6_VERTSTRIDE_32] = 32,
      [7] = 64,
      [8] = 128,
      [9] = 256,
      [GEN6_VERTSTRIDE_VXH] = 0,
   };
   const int exec_size = exec_size_map[cg->inst->exec_size];
   int i;

   /* Sandy Bridge PRM, volume 4 part 2, page 94 */

   /* 1. (we don't do 32 anyway) */
   assert(exec_size <= 16);

   for (i = 0; i < ARRAY_SIZE(cg->src); i++) {
      const int width = width_map[cg->src[i].width];
      const int horz_stride = horz_stride_map[cg->src[i].horz_stride];
      const int vert_stride = vert_stride_map[cg->src[i].vert_stride];

      if (src_is_null(cg, i))
         break;

      /* 3. */
      assert(exec_size >= width);

      if (exec_size == width) {
         /* 4. & 5. */
         if (horz_stride)
            assert(vert_stride == width * horz_stride);
      }

      if (width == 1) {
         /* 6. */
         assert(horz_stride == 0);

         /* 7. */
         if (exec_size == 1)
            assert(vert_stride == 0);
      }

      /* 8. */
      if (!vert_stride && !horz_stride)
         assert(width == 1);
   }

   /* derived from 10.1.2. & 10.2. */
   assert(cg->dst.horz_stride != GEN6_HORZSTRIDE_0);
}

static unsigned
translate_vfile(enum toy_file file)
{
   switch (file) {
   case TOY_FILE_ARF:   return GEN6_FILE_ARF;
   case TOY_FILE_GRF:   return GEN6_FILE_GRF;
   case TOY_FILE_MRF:   return GEN6_FILE_MRF;
   case TOY_FILE_IMM:   return GEN6_FILE_IMM;
   default:
      assert(!"unhandled toy file");
      return GEN6_FILE_GRF;
   }
}

static unsigned
translate_vtype(enum toy_type type)
{
   switch (type) {
   case TOY_TYPE_F:     return GEN6_TYPE_F;
   case TOY_TYPE_D:     return GEN6_TYPE_D;
   case TOY_TYPE_UD:    return GEN6_TYPE_UD;
   case TOY_TYPE_W:     return GEN6_TYPE_W;
   case TOY_TYPE_UW:    return GEN6_TYPE_UW;
   case TOY_TYPE_V:     return GEN6_TYPE_V_IMM;
   default:
      assert(!"unhandled toy type");
      return GEN6_TYPE_F;
   }
}

static unsigned
translate_writemask(enum toy_writemask writemask)
{
   /* TOY_WRITEMASK_* are compatible with the hardware definitions */
   assert(writemask <= 0xf);
   return writemask;
}

static unsigned
translate_swizzle(enum toy_swizzle swizzle)
{
   /* TOY_SWIZZLE_* are compatible with the hardware definitions */
   assert(swizzle <= 3);
   return swizzle;
}

/**
 * Prepare for generating an instruction.
 */
static void
codegen_prepare(struct codegen *cg, const struct ilo_dev *dev,
                const struct toy_inst *inst, int pc, int rect_linear_width)
{
   int i;

   cg->dev = dev;
   cg->inst = inst;
   cg->pc = pc;

   cg->flag_reg_num = 0;
   cg->flag_sub_reg_num = 0;

   cg->dst.file = translate_vfile(inst->dst.file);
   cg->dst.type = translate_vtype(inst->dst.type);
   cg->dst.indirect = inst->dst.indirect;
   cg->dst.indirect_subreg = inst->dst.indirect_subreg;
   cg->dst.origin = inst->dst.val32;

   /*
    * From the Sandy Bridge PRM, volume 4 part 2, page 81:
    *
    *     "For a word or an unsigned word immediate data, software must
    *      replicate the same 16-bit immediate value to both the lower word
    *      and the high word of the 32-bit immediate field in an instruction."
    */
   if (inst->dst.file == TOY_FILE_IMM) {
      switch (inst->dst.type) {
      case TOY_TYPE_W:
      case TOY_TYPE_UW:
         cg->dst.origin &= 0xffff;
         cg->dst.origin |= cg->dst.origin << 16;
         break;
      default:
         break;
      }
   }

   cg->dst.writemask = translate_writemask(inst->dst.writemask);

   switch (inst->dst.rect) {
   case TOY_RECT_LINEAR:
      cg->dst.horz_stride = GEN6_HORZSTRIDE_1;
      break;
   default:
      assert(!"unsupported dst region");
      cg->dst.horz_stride = GEN6_HORZSTRIDE_1;
      break;
   }

   for (i = 0; i < ARRAY_SIZE(cg->src); i++) {
      struct codegen_src *src = &cg->src[i];

      src->file = translate_vfile(inst->src[i].file);
      src->type = translate_vtype(inst->src[i].type);
      src->indirect = inst->src[i].indirect;
      src->indirect_subreg = inst->src[i].indirect_subreg;
      src->origin = inst->src[i].val32;

      /* do the same for src */
      if (inst->dst.file == TOY_FILE_IMM) {
         switch (inst->src[i].type) {
         case TOY_TYPE_W:
         case TOY_TYPE_UW:
            src->origin &= 0xffff;
            src->origin |= src->origin << 16;
            break;
         default:
            break;
         }
      }

      src->swizzle[0] = translate_swizzle(inst->src[i].swizzle_x);
      src->swizzle[1] = translate_swizzle(inst->src[i].swizzle_y);
      src->swizzle[2] = translate_swizzle(inst->src[i].swizzle_z);
      src->swizzle[3] = translate_swizzle(inst->src[i].swizzle_w);
      src->absolute = inst->src[i].absolute;
      src->negate = inst->src[i].negate;

      switch (inst->src[i].rect) {
      case TOY_RECT_LINEAR:
         switch (rect_linear_width) {
         case 1:
            src->vert_stride = GEN6_VERTSTRIDE_1;
            src->width = GEN6_WIDTH_1;
            break;
         case 2:
            src->vert_stride = GEN6_VERTSTRIDE_2;
            src->width = GEN6_WIDTH_2;
            break;
         case 4:
            src->vert_stride = GEN6_VERTSTRIDE_4;
            src->width = GEN6_WIDTH_4;
            break;
         case 8:
            src->vert_stride = GEN6_VERTSTRIDE_8;
            src->width = GEN6_WIDTH_8;
            break;
         case 16:
            src->vert_stride = GEN6_VERTSTRIDE_16;
            src->width = GEN6_WIDTH_16;
            break;
         default:
            assert(!"unsupported TOY_RECT_LINEAR width");
            src->vert_stride = GEN6_VERTSTRIDE_1;
            src->width = GEN6_WIDTH_1;
            break;
         }
         src->horz_stride = GEN6_HORZSTRIDE_1;
         break;
      case TOY_RECT_041:
         src->vert_stride = GEN6_VERTSTRIDE_0;
         src->width = GEN6_WIDTH_4;
         src->horz_stride = GEN6_HORZSTRIDE_1;
         break;
      case TOY_RECT_010:
         src->vert_stride = GEN6_VERTSTRIDE_0;
         src->width = GEN6_WIDTH_1;
         src->horz_stride = GEN6_HORZSTRIDE_0;
         break;
      case TOY_RECT_220:
         src->vert_stride = GEN6_VERTSTRIDE_2;
         src->width = GEN6_WIDTH_2;
         src->horz_stride = GEN6_HORZSTRIDE_0;
         break;
      case TOY_RECT_440:
         src->vert_stride = GEN6_VERTSTRIDE_4;
         src->width = GEN6_WIDTH_4;
         src->horz_stride = GEN6_HORZSTRIDE_0;
         break;
      case TOY_RECT_240:
         src->vert_stride = GEN6_VERTSTRIDE_2;
         src->width = GEN6_WIDTH_4;
         src->horz_stride = GEN6_HORZSTRIDE_0;
         break;
      default:
         assert(!"unsupported src region");
         src->vert_stride = GEN6_VERTSTRIDE_1;
         src->width = GEN6_WIDTH_1;
         src->horz_stride = GEN6_HORZSTRIDE_1;
         break;
      }
   }
}

/**
 * Generate HW shader code.  The instructions should have been legalized.
 */
void *
toy_compiler_assemble(struct toy_compiler *tc, int *size)
{
   const struct toy_inst *inst;
   uint32_t *code;
   int pc;

   code = MALLOC(tc->num_instructions * 4 * sizeof(uint32_t));
   if (!code)
      return NULL;

   pc = 0;
   tc_head(tc);
   while ((inst = tc_next(tc)) != NULL) {
      uint32_t *dw = &code[pc * 4];
      struct codegen cg;

      if (pc >= tc->num_instructions) {
         tc_fail(tc, "wrong instructoun count");
         break;
      }

      codegen_prepare(&cg, tc->dev, inst, pc, tc->rect_linear_width);
      codegen_validate_region_restrictions(&cg);

      switch (inst->opcode) {
      case GEN6_OPCODE_MAD:
         codegen_inst_3src_gen6(&cg, dw);
         break;
      default:
         codegen_inst_gen6(&cg, dw);
         break;
      }

      pc++;
   }

   /* never return an invalid kernel */
   if (tc->fail) {
      FREE(code);
      return NULL;
   }

   if (size)
      *size = pc * 4 * sizeof(uint32_t);

   return code;
}
