src/thirdparty/VirtualDub/system/source/debugx86.cpp

   1 //      VirtualDub - Video processing and capture application
   2 //      System library component
   3 //      Copyright (C) 1998-2004 Avery Lee, All Rights Reserved.
   4 //
   5 //      Beginning with 1.6.0, the VirtualDub system library is licensed
   6 //      differently than the remainder of VirtualDub.  This particular file is
   7 //      thus licensed as follows (the "zlib" license):
   8 //
   9 //      This software is provided 'as-is', without any express or implied
  10 //      warranty.  In no event will the authors be held liable for any
  11 //      damages arising from the use of this software.
  12 //
  13 //      Permission is granted to anyone to use this software for any purpose,
  14 //      including commercial applications, and to alter it and redistribute it
  15 //      freely, subject to the following restrictions:
  16 //
  17 //      1.      The origin of this software must not be misrepresented; you must
  18 //              not claim that you wrote the original software. If you use this
  19 //              software in a product, an acknowledgment in the product
  20 //              documentation would be appreciated but is not required.
  21 //      2.      Altered source versions must be plainly marked as such, and must
  22 //              not be misrepresented as being the original software.
  23 //      3.      This notice may not be removed or altered from any source
  24 //              distribution.
  25
  26 #include "stdafx.h"
  27 #include <vd2/system/seh.h>
  28 #include <vd2/system/vdtypes.h>
  29 #include <vd2/system/debugx86.h>
  30
  31 bool VDIsValidCallX86(const char *buf, int len) {
  32         // Permissible CALL sequences that we care about:
  33         //
  34         //      E8 xx xx xx xx                  CALL near relative
  35         //      FF (group 2)                    CALL near absolute indirect
  36         //
  37         // Minimum sequence is 2 bytes (call eax).
  38         // Maximum sequence is 7 bytes (call dword ptr [eax+disp32]).
  39
  40         if (len >= 5 && buf[-5] == (char)0xE8)
  41                 return true;
  42
  43         // FF 14 xx                                     CALL [reg32+reg32*scale]
  44
  45         if (len >= 3 && buf[-3] == (char)0xFF && buf[-2]==0x14)
  46                 return true;
  47
  48         // FF 15 xx xx xx xx            CALL disp32
  49
  50         if (len >= 6 && buf[-6] == (char)0xFF && buf[-5]==0x15)
  51                 return true;
  52
  53         // FF 00-3F(!14/15)                     CALL [reg32]
  54
  55         if (len >= 2 && buf[-2] == (char)0xFF && (unsigned char)buf[-1] < 0x40)
  56                 return true;
  57
  58         // FF D0-D7                                     CALL reg32
  59
  60         if (len >= 2 && buf[-2] == (char)0xFF && (buf[-1]&0xF8) == 0xD0)
  61                 return true;
  62
  63         // FF 50-57 xx                          CALL [reg32+reg32*scale+disp8]
  64
  65         if (len >= 3 && buf[-3] == (char)0xFF && (buf[-2]&0xF8) == 0x50)
  66                 return true;
  67
  68         // FF 90-97 xx xx xx xx xx      CALL [reg32+reg32*scale+disp32]
  69
  70         if (len >= 7 && buf[-7] == (char)0xFF && (buf[-6]&0xF8) == 0x90)
  71                 return true;
  72
  73         return false;
  74 }
  75
  76 VDInstructionTypeX86 VDGetInstructionTypeX86(const void *p) {
  77         struct local {
  78                 static bool RangeHitTest(const uint8 *range, uint8 c) {
  79                         while(*range) {
  80                                 if (c>=range[0] && c<=range[1])
  81                                         return true;
  82                                 range += 2;
  83                         }
  84
  85                         return false;
  86                 }
  87         };
  88
  89         VDInstructionTypeX86 type = kX86InstUnknown;
  90
  91         vd_seh_guard_try {
  92                 unsigned char buf[8];
  93
  94                 memcpy(buf, p, 8);
  95
  96                 if (buf[0] == 0x0f && buf[1] == 0x0f)
  97                         type = kX86Inst3DNow;                   // Conveniently, all 3DNow! instructions begin 0F 0F
  98                 else if ((buf[0] == 0xdb || buf[0] == 0xdf) && (buf[1]>=0xe8 && buf[1]<=0xf7))
  99                         type = kX86InstP6;                              // DB/DF E8-F7: FCOMI/FCOMIP/FUCOMI/FUCOMIP (P6)
 100                 else if ((buf[0]&0xfe)==0xda && (buf[1]&0xe0)==0xc0)
 101                         type = kX86InstP6;                              // DA/DB C0-DF: FCMOVcc (P6)
 102                 else if (buf[0] == 0x0f && (buf[1]&0xf0)==0x40)
 103                         type = kX86InstP6;                              // 0F 40-4F: CMOVcc (P6)
 104                 else {
 105                         const unsigned char *s = buf;
 106                         bool bWide = false;
 107                         bool bRepF2 = false;
 108                         bool bRepF3 = false;
 109
 110                         // At this point we're down to MMX, SSE, SSE2 -- which makes things simpler
 111                         // as we must see F2 0F, F3 0F, or 0F next.  MMX ops use 0F exclusively,
 112                         // some SSE ops use F2, and a few SSE2 ones use F3.  If we see 66 on an
 113                         // MMX or SSE op it's automatically SSE2 as it's either a 128-bit MMX op
 114                         // or a double-precision version of an SSE one.
 115
 116                         if (*s == 0x66) {               // 66h override used by SSE2 and is supposed to be ahead of F2/F3 in encodings
 117                                 ++s;
 118                                 bWide = true;
 119                         }
 120
 121                         if (*s == 0xf2) {
 122                                 ++s;
 123                                 bRepF2 = true;
 124                         }
 125
 126                         if (*s == 0xf3) {
 127                                 ++s;
 128                                 bRepF3 = true;
 129                         }
 130
 131                         if (*s++ == 0x0f) {
 132                                 // SSE - 1x, 28-2F, 5x, C2, AE
 133                                 // MMX2 - 70, C4-C6, D7, DA, DE, E0, E3, E4, E7, EA, EE, F6, F7
 134                                 // MMX - 6x, 7x, Dx, Ex, and Fx except for MMX2
 135                                 // SSE2 - C3, SSE ops with 66 or F2, MMX/MMX2 ops with 66/F2/F3
 136
 137                                 static const uint8 sse_ranges[]={0x10,0x1f,0x28,0x2f,0x50,0x5f,0xc2,0xc2,0xae,0xae,0};
 138                                 static const uint8 sse2_ranges[]={0xc3,0xc3,0};
 139                                 static const uint8 mmx2_ranges[]={0x70,0x70,0xc4,0xc6,0xd7,0xd7,0xda,0xda,0xde,0xde,0xe0,0xe0,0xe3,0xe4,0xe7,0xe7,0xea,0xea,0xee,0xee,0xf6,0xf7,0};
 140                                 static const uint8 mmx_ranges[]={0x60,0x7f,0xd0,0xff,0};
 141
 142                                 if (local::RangeHitTest(sse_ranges, *s))
 143                                         type = (bWide||bRepF2) ? kX86InstSSE2 : kX86InstSSE;
 144                                 else if (local::RangeHitTest(sse2_ranges, *s))
 145                                         type = kX86InstSSE2;
 146                                 else if (local::RangeHitTest(mmx2_ranges, *s))
 147                                         type = (bWide||bRepF2||bRepF3) ? kX86InstSSE2 : kX86InstMMX2;
 148                                 else if (local::RangeHitTest(mmx_ranges, *s))
 149                                         type = (bWide||bRepF2||bRepF3) ? kX86InstSSE2 : kX86InstMMX;
 150                         }
 151                 }
 152         } vd_seh_guard_except {
 153         }
 154
 155         return type;
 156 }